From b42a68ae2cab6a6ca750b0f8dfc41cc4b179b895 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Fri, 10 Apr 2026 15:35:27 -0600 Subject: [PATCH 01/10] feat: offline disk overflow for network-aware event persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the memory queue reaches capacity while offline, evicted events now overflow to a batched in-memory buffer that flushes to a separate disk file (via the native bridge). On reconnect, two independent flush paths run: the memory queue flushes normally, and overflow events drain directly from disk to network in batches of 100 — avoiding loading 10K events into a 2K memory queue. Key changes: - Dispatcher: onOverflow callback redirects evicted events, sendBatchDirect bypasses the queue for disk-to-network drain - PersistentEventQueue: handleOverflow batched buffer, flushOverflowBufferToDisk, drainDiskToNetwork with TTL filtering - NativeQueueStorage: overflow snapshot read/write/delete methods - MetaRouterAnalyticsClient: wires overflow callback, dual flush on reconnect, disk drain on launch, maxOfflineDiskEvents passthrough --- .../MetaRouterAnalyticsClient.test.ts | 46 +++ src/analytics/MetaRouterAnalyticsClient.ts | 13 +- src/analytics/dispatcher.test.ts | 130 ++++++ src/analytics/dispatcher.ts | 57 ++- .../persistence/NativeQueueStorage.ts | 41 ++ .../persistence/PersistentEventQueue.ts | 212 +++++++++- .../__tests__/PersistentEventQueue.test.ts | 386 +++++++++++++++++- .../persistence/__tests__/integration.test.ts | 95 ++++- src/analytics/persistence/types.ts | 6 + 9 files changed, 970 insertions(+), 16 deletions(-) diff --git a/src/analytics/MetaRouterAnalyticsClient.test.ts b/src/analytics/MetaRouterAnalyticsClient.test.ts index 472434d..e42b4cc 100644 --- a/src/analytics/MetaRouterAnalyticsClient.test.ts +++ b/src/analytics/MetaRouterAnalyticsClient.test.ts @@ -674,6 +674,52 @@ describe('MetaRouterAnalyticsClient', () => { await client.reset(); expect(stopSpy).toHaveBeenCalled(); }); + + it('overflow events redirect to persistent queue on memory cap exceeded', async () => { + const monitor = new StubNetworkMonitor('disconnected'); + const client = new MetaRouterAnalyticsClient( + { + ...opts, + maxQueueBytes: 500, // very small cap + flushIntervalSeconds: 3600, + }, + { networkMonitor: monitor } + ); + await client.init(); + + // Override to prevent auto-flush from interfering + jest.spyOn(client as any, 'flush').mockResolvedValue(undefined); + (client as any).dispatcher.opts.autoFlushThreshold = 9999; + + // Track enough events to overflow memory queue + for (let i = 0; i < 30; i++) { + client.track(`event_${i}`, { data: 'x'.repeat(20) }); + } + + // Check that overflow buffer has events + expect( + (client as any).persistentQueue.overflowBufferCount + ).toBeGreaterThan(0); + }); + + it('offline -> online drains disk overflow', async () => { + const monitor = new StubNetworkMonitor('connected'); + const client = new MetaRouterAnalyticsClient(opts, { + networkMonitor: monitor, + }); + await client.init(); + + // Spy on drainDiskToNetwork + const drainSpy = jest + .spyOn((client as any).persistentQueue, 'drainDiskToNetwork') + .mockResolvedValue(undefined); + + // Go offline then online + monitor.simulate('disconnected'); + monitor.simulate('connected'); + + expect(drainSpy).toHaveBeenCalled(); + }); }); describe('tracing', () => { diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index 066451d..1f974b8 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -121,6 +121,7 @@ export class MetaRouterAnalyticsClient { log, warn, error, + onOverflow: (events) => this.persistentQueue.handleOverflow(events), onScheduleFlushIn: (ms) => { (this as any).scheduleFlushIn(ms, { fromDispatcher: true }); }, @@ -130,7 +131,9 @@ export class MetaRouterAnalyticsClient { }); this.queue = this.dispatcher.getQueueRef(); - this.persistentQueue = new PersistentEventQueue(this.dispatcher); + this.persistentQueue = new PersistentEventQueue(this.dispatcher, { + maxOfflineDiskEvents: options.maxOfflineDiskEvents, + }); } /** @@ -183,7 +186,10 @@ export class MetaRouterAnalyticsClient { if (wasOffline && status === 'connected') { log('Network connectivity restored — resuming flush'); this.dispatcher.resetCircuitBreaker(); + // (1) Memory queue → network (existing path) void this.flush(); + // (2) Disk overflow → network directly (no rehydration into memory queue) + void this.persistentQueue.drainDiskToNetwork(this.dispatcher); } else if (status === 'disconnected') { log('Network connectivity lost — pausing HTTP attempts'); } @@ -195,6 +201,11 @@ export class MetaRouterAnalyticsClient { // Flush immediately so rehydrated events ship on cold start // (AppState 'change' won't fire because the app is already active) void this.flush(); + + // If online at launch, drain any overflow from a previous session + if (this.networkStatus === 'connected') { + void this.persistentQueue.drainDiskToNetwork(this.dispatcher); + } } catch (error) { this.lifecycle = 'idle'; // allow retry warn('Analytics client initialization failed:', error); diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 94bf853..11b19b5 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -427,6 +427,136 @@ describe('Dispatcher', () => { }); }); + describe('onOverflow callback', () => { + it('fires with events evicted from queue instead of dropping', () => { + const opts = baseOpts(); + const overflowEvents: any[][] = []; + opts.onOverflow = jest.fn((events: any[]) => { + overflowEvents.push(events); + }); + opts.maxQueueBytes = 84; // fits ~3 small events + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + d.enqueue({ type: 'track', event: 'c' } as any); + d.enqueue({ type: 'track', event: 'd' } as any); // evicts oldest + + expect(opts.onOverflow).toHaveBeenCalled(); + expect(overflowEvents[0].length).toBeGreaterThanOrEqual(1); + expect(overflowEvents[0][0].event).toBe('a'); + + // warn should NOT be called with "dropped oldest" when onOverflow is set + expect( + (opts.warn as jest.Mock).mock.calls.some((c) => + String(c[0]).includes('Queue cap reached') + ) + ).toBe(false); + }); + + it('does not fire when queue is within capacity', () => { + const opts = baseOpts(); + opts.onOverflow = jest.fn(); + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + expect(opts.onOverflow).not.toHaveBeenCalled(); + }); + + it('batches multiple evictions into a single callback', () => { + const opts = baseOpts(); + const overflowBatches: any[][] = []; + opts.onOverflow = jest.fn((events: any[]) => { + overflowBatches.push([...events]); + }); + // Set capacity to fit exactly 1 event + const eventSize = new TextEncoder().encode( + JSON.stringify({ type: 'track', event: 'x' }) + ).byteLength; + opts.maxQueueBytes = eventSize; + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); // evicts 'a' + d.enqueue({ type: 'track', event: 'c' } as any); // evicts 'b' + + // Each enqueue that causes eviction fires a callback + expect(opts.onOverflow).toHaveBeenCalledTimes(2); + }); + }); + + describe('sendBatchDirect', () => { + it('returns true on successful send', async () => { + const opts = baseOpts(); + const d = new Dispatcher(opts); + + const result = await d.sendBatchDirect([ + { type: 'track', event: 'e1' } as any, + ]); + expect(result).toBe(true); + expect(opts.fetchWithTimeout).toHaveBeenCalled(); + }); + + it('returns false on HTTP error', async () => { + const opts = baseOpts(); + opts.fetchWithTimeout = jest.fn(async () => ({ + ok: false, + status: 500, + })) as any; + const d = new Dispatcher(opts); + + const result = await d.sendBatchDirect([ + { type: 'track', event: 'e1' } as any, + ]); + expect(result).toBe(false); + }); + + it('returns false on network error', async () => { + const opts = baseOpts(); + opts.fetchWithTimeout = jest.fn(async () => { + throw new Error('Network unavailable'); + }); + const d = new Dispatcher(opts); + + const result = await d.sendBatchDirect([ + { type: 'track', event: 'e1' } as any, + ]); + expect(result).toBe(false); + }); + + it('does not affect the memory queue', async () => { + const opts = baseOpts(); + opts.autoFlushThreshold = 9999; + const d = new Dispatcher(opts); + + d.enqueue({ type: 'track', event: 'queued' } as any); + await d.sendBatchDirect([{ type: 'track', event: 'direct' } as any]); + + // Memory queue unchanged + expect(d.getQueueRef().length).toBe(1); + expect((d.getQueueRef()[0] as any).event).toBe('queued'); + }); + + it('stamps sentAt on batch events', async () => { + const opts = baseOpts(); + let sentBody: any = null; + opts.fetchWithTimeout = jest.fn(async (_url?: string, init?: any) => { + sentBody = JSON.parse(init.body); + return { ok: true, status: 200 } as any; + }); + const d = new Dispatcher(opts); + + await d.sendBatchDirect([{ type: 'track', event: 'e1' } as any]); + + expect(sentBody.batch[0].sentAt).toBeDefined(); + }); + }); + it('stress test: all ~2K queued events successfully transmit in batches', async () => { const opts = baseOpts(); opts.autoFlushThreshold = 9999; // disable auto-flush diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index ee4ee67..e71edff 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -28,6 +28,7 @@ export interface DispatcherOptions { warn: (...args: any[]) => void; error: (...args: any[]) => void; + onOverflow?: (events: EventPayload[]) => void; // called with events evicted from memory queue when cap is hit onScheduleFlushIn?: (ms: number) => void; // optional notification for tests/metrics onFatalConfig?: () => void; // 401/403/404 handler } @@ -129,14 +130,24 @@ export default class Dispatcher { enqueue(event: EventPayload): void { const eventSize = this.estimateEventSize(event); - // Hard cap: drop oldest until there's room (byte limit) + // Hard cap: evict oldest until there's room (byte limit) + const overflowBatch: EventPayload[] = []; while ( this.queue.length > 0 && this.queueSizeBytes + eventSize > this.opts.maxQueueBytes ) { const dropped = this.queue.shift(); - if (dropped) this.queueSizeBytes -= this.estimateEventSize(dropped); - this.opts.warn('Queue cap reached — dropped oldest event'); + if (dropped) { + this.queueSizeBytes -= this.estimateEventSize(dropped); + if (this.opts.onOverflow) { + overflowBatch.push(dropped); + } else { + this.opts.warn('Queue cap reached — dropped oldest event'); + } + } + } + if (overflowBatch.length > 0) { + this.opts.onOverflow!(overflowBatch); } this.opts.log( @@ -394,6 +405,46 @@ export default class Dispatcher { this.consecutiveRetries = 0; } + /** + * Send a batch directly to network, bypassing the memory queue. + * Used by disk drain to flush overflow without loading into queue. + * Returns true on success, false on failure. + */ + async sendBatchDirect(events: EventPayload[]): Promise { + try { + const nowIso = new Date().toISOString(); + const batch = events.map((e) => ({ ...(e as any), sentAt: nowIso })); + + const headers: Record = { + 'Content-Type': 'application/json', + }; + if (this.opts.isTracingEnabled()) { + headers.Trace = 'true'; + } + + const res = await this.opts.fetchWithTimeout( + this.opts.endpoint('/v1/batch'), + { + method: 'POST', + headers, + body: JSON.stringify({ batch }), + }, + 8000 + ); + + if (res.ok) { + this.opts.log(`Direct batch send successful (${events.length} events)`); + return true; + } + + this.opts.warn(`Direct batch send failed with status ${res.status}`); + return false; + } catch (err) { + this.opts.warn(`Direct batch send failed: ${(err as any)?.message}`); + return false; + } + } + getDebugInfo() { return { queueLength: this.queue.length, diff --git a/src/analytics/persistence/NativeQueueStorage.ts b/src/analytics/persistence/NativeQueueStorage.ts index 3d73a1a..09db7dd 100644 --- a/src/analytics/persistence/NativeQueueStorage.ts +++ b/src/analytics/persistence/NativeQueueStorage.ts @@ -8,11 +8,19 @@ import { warn } from '../utils/logger'; * - readSnapshot(): Promise — returns file contents or null * - writeSnapshot(data: string): Promise — overwrites file with data * - deleteSnapshot(): Promise — deletes file if it exists + * + * Overflow snapshot methods (for offline disk overflow): + * - readOverflowSnapshot(): Promise + * - writeOverflowSnapshot(data: string): Promise + * - deleteOverflowSnapshot(): Promise */ interface NativeQueueStorageModule { readSnapshot(): Promise; writeSnapshot(data: string): Promise; deleteSnapshot(): Promise; + readOverflowSnapshot?(): Promise; + writeOverflowSnapshot?(data: string): Promise; + deleteOverflowSnapshot?(): Promise; } function getModule(): NativeQueueStorageModule | null { @@ -58,3 +66,36 @@ export async function deleteSnapshot(): Promise { warn('Failed to delete queue snapshot from disk:', err); } } + +// --- Overflow snapshot (offline disk overflow) --- + +export async function readOverflowSnapshot(): Promise { + const mod = getModule(); + if (!mod?.readOverflowSnapshot) return null; + try { + return await mod.readOverflowSnapshot(); + } catch (err) { + warn('Failed to read overflow snapshot from disk:', err); + return null; + } +} + +export async function writeOverflowSnapshot(data: string): Promise { + const mod = getModule(); + if (!mod?.writeOverflowSnapshot) return; + try { + await mod.writeOverflowSnapshot(data); + } catch (err) { + warn('Failed to write overflow snapshot to disk:', err); + } +} + +export async function deleteOverflowSnapshot(): Promise { + const mod = getModule(); + if (!mod?.deleteOverflowSnapshot) return; + try { + await mod.deleteOverflowSnapshot(); + } catch (err) { + warn('Failed to delete overflow snapshot from disk:', err); + } +} diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index b8e07e4..3efb71b 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -1,14 +1,20 @@ +import type { EventPayload } from '../types'; import type Dispatcher from '../dispatcher'; import { readSnapshot, writeSnapshot, deleteSnapshot as nativeDeleteSnapshot, + readOverflowSnapshot, + writeOverflowSnapshot, + deleteOverflowSnapshot as nativeDeleteOverflowSnapshot, } from './NativeQueueStorage'; import { SNAPSHOT_VERSION, FLUSH_THRESHOLD_BYTES, FLUSH_THRESHOLD_COUNT, DEFAULT_EVENT_TTL_MS, + DEFAULT_MAX_OFFLINE_DISK_EVENTS, + OVERFLOW_BATCH_THRESHOLD, type QueueSnapshot, } from './types'; import { log, warn } from '../utils/logger'; @@ -24,9 +30,18 @@ export class PersistentEventQueue { private readonly dispatcher: Dispatcher; private flushInFlight: Promise | null = null; private _rehydratedEvents: number = 0; + private readonly maxOfflineDiskEvents: number; + private overflowBuffer: EventPayload[] = []; + private overflowFlushInFlight: Promise | null = null; + private drainInFlight: Promise | null = null; - constructor(dispatcher: Dispatcher) { + constructor( + dispatcher: Dispatcher, + opts?: { maxOfflineDiskEvents?: number } + ) { this.dispatcher = dispatcher; + this.maxOfflineDiskEvents = + opts?.maxOfflineDiskEvents ?? DEFAULT_MAX_OFFLINE_DISK_EVENTS; } get rehydratedEvents(): number { @@ -126,17 +141,21 @@ export class PersistentEventQueue { if (queue.length === 0) { log('Queue empty — deleting snapshot'); await nativeDeleteSnapshot(); - return; - } + } else { + const snapshot: QueueSnapshot = { + version: SNAPSHOT_VERSION, + events: [...queue], // shallow copy to avoid mutation during async write + }; - const snapshot: QueueSnapshot = { - version: SNAPSHOT_VERSION, - events: [...queue], // shallow copy to avoid mutation during async write - }; + const data = JSON.stringify(snapshot); + await writeSnapshot(data); + log(`Queue snapshot written to disk (${queue.length} events)`); + } - const data = JSON.stringify(snapshot); - await writeSnapshot(data); - log(`Queue snapshot written to disk (${queue.length} events)`); + // Also flush overflow buffer to disk on background + if (this.overflowBuffer.length > 0) { + await this.flushOverflowBufferToDisk(); + } } catch (err) { warn('Failed to flush queue to disk:', err); } @@ -152,11 +171,184 @@ export class PersistentEventQueue { ); } + /** + * Handle events evicted from the memory queue (overflow callback). + * Buffers events and flushes to disk in batches. + */ + handleOverflow(events: EventPayload[]): void { + this.overflowBuffer.push(...events); + + // Enforce disk cap on buffer: drop oldest if overflow exceeds max + while (this.overflowBuffer.length > this.maxOfflineDiskEvents) { + this.overflowBuffer.shift(); + warn('Offline disk cap reached — dropped oldest overflow event'); + } + + if (this.overflowBuffer.length >= OVERFLOW_BATCH_THRESHOLD) { + void this.flushOverflowBufferToDisk(); + } + } + + /** + * Flush the in-memory overflow buffer to the overflow disk store. + * Singleflight: concurrent calls coalesce into one write. + */ + async flushOverflowBufferToDisk(): Promise { + if (this.overflowFlushInFlight) return this.overflowFlushInFlight; + + this.overflowFlushInFlight = this._doFlushOverflowToDisk().finally(() => { + this.overflowFlushInFlight = null; + }); + return this.overflowFlushInFlight; + } + + private async _doFlushOverflowToDisk(): Promise { + if (this.overflowBuffer.length === 0) return; + + try { + const batch = this.overflowBuffer.splice(0); + + // Read existing overflow on disk + const raw = await readOverflowSnapshot(); + let existing: EventPayload[] = []; + if (raw) { + try { + const snapshot: QueueSnapshot = JSON.parse(raw); + if ( + snapshot.version === SNAPSHOT_VERSION && + Array.isArray(snapshot.events) + ) { + existing = snapshot.events; + } + } catch { + warn('Overflow snapshot is corrupt JSON — overwriting'); + } + } + + let combined = existing.concat(batch); + + // Enforce disk cap + if (combined.length > this.maxOfflineDiskEvents) { + const dropCount = combined.length - this.maxOfflineDiskEvents; + combined = combined.slice(dropCount); + warn( + `Offline overflow disk cap enforced — dropped ${dropCount} oldest events` + ); + } + + const snapshot: QueueSnapshot = { + version: SNAPSHOT_VERSION, + events: combined, + }; + await writeOverflowSnapshot(JSON.stringify(snapshot)); + log( + `Overflow buffer flushed to disk (${batch.length} new, ${combined.length} total on disk)` + ); + } catch (err) { + warn('Failed to flush overflow buffer to disk:', err); + } + } + + /** + * Drain overflow events directly from disk to network in batches. + * Does NOT load events into the memory queue. + * Called on offline→online transition. + */ + async drainDiskToNetwork(dispatcher: Dispatcher): Promise { + if (this.drainInFlight) return this.drainInFlight; + + this.drainInFlight = this._doDrainDiskToNetwork(dispatcher).finally(() => { + this.drainInFlight = null; + }); + return this.drainInFlight; + } + + private async _doDrainDiskToNetwork(dispatcher: Dispatcher): Promise { + try { + // First, flush any remaining buffer to disk + await this.flushOverflowBufferToDisk(); + + while (true) { + const raw = await readOverflowSnapshot(); + if (!raw) return; + + let snapshot: QueueSnapshot; + try { + snapshot = JSON.parse(raw); + } catch { + warn('Overflow snapshot is corrupt during drain — deleting'); + await nativeDeleteOverflowSnapshot(); + return; + } + + if (!Array.isArray(snapshot.events) || snapshot.events.length === 0) { + await nativeDeleteOverflowSnapshot(); + return; + } + + // Filter expired events + const now = Date.now(); + const cutoff = now - DEFAULT_EVENT_TTL_MS; + const events = snapshot.events.filter((e) => { + const ts = (e as any).timestamp; + if (!ts) return true; + const eventTime = new Date(ts).getTime(); + return !isNaN(eventTime) && eventTime > cutoff; + }); + + if (events.length === 0) { + log('All overflow events expired — deleting'); + await nativeDeleteOverflowSnapshot(); + return; + } + + // Take a batch from the front (oldest first) + const batchSize = Math.min(100, events.length); + const batch = events.slice(0, batchSize); + const remaining = events.slice(batchSize); + + const success = await dispatcher.sendBatchDirect(batch); + if (success) { + if (remaining.length === 0) { + await nativeDeleteOverflowSnapshot(); + log('Overflow disk drain complete'); + } else { + const updated: QueueSnapshot = { + version: SNAPSHOT_VERSION, + events: remaining, + }; + await writeOverflowSnapshot(JSON.stringify(updated)); + log( + `Overflow drain batch sent (${batch.length}), ${remaining.length} remaining on disk` + ); + } + } else { + // Network failed — stop draining, will retry on next online transition + log( + `Overflow drain paused — network send failed, ${events.length} events remain on disk` + ); + return; + } + } + } catch (err) { + warn('Failed to drain overflow from disk:', err); + } + } + + /** + * Get the number of events in the overflow buffer (not yet flushed to disk). + */ + get overflowBufferCount(): number { + return this.overflowBuffer.length; + } + /** * Delete the disk snapshot (used during reset). */ async deleteSnapshot(): Promise { await nativeDeleteSnapshot(); + await nativeDeleteOverflowSnapshot(); + this.overflowBuffer.length = 0; } } diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 6a05c57..106794b 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -4,7 +4,10 @@ import CircuitBreaker from '../../utils/circuitBreaker'; // Helpers function mockNativeStorage() { - const store: { data: string | null } = { data: null }; + const store: { data: string | null; overflow: string | null } = { + data: null, + overflow: null, + }; NativeModules.MetaRouterQueueStorage = { readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { @@ -13,6 +16,13 @@ function mockNativeStorage() { deleteSnapshot: jest.fn(async () => { store.data = null; }), + readOverflowSnapshot: jest.fn(async () => store.overflow), + writeOverflowSnapshot: jest.fn(async (data: string) => { + store.overflow = data; + }), + deleteOverflowSnapshot: jest.fn(async () => { + store.overflow = null; + }), }; return { store, mock: NativeModules.MetaRouterQueueStorage }; } @@ -544,6 +554,380 @@ describe('PersistentEventQueue', () => { }); }); + describe('handleOverflow', () => { + it('accumulates events in overflow buffer', () => { + mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.handleOverflow([ + { type: 'track', event: 'o1' }, + { type: 'track', event: 'o2' }, + ]); + + expect(pq.overflowBufferCount).toBe(2); + }); + + it('respects maxOfflineDiskEvents cap on buffer', () => { + mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher, { + maxOfflineDiskEvents: 5, + }); + + const events = Array.from({ length: 10 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })); + pq.handleOverflow(events); + + // Buffer should be capped at 5 (oldest dropped) + expect(pq.overflowBufferCount).toBe(5); + }); + + it('triggers flush to disk at batch threshold', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + // Overflow exactly 100 events (the OVERFLOW_BATCH_THRESHOLD) + const events = Array.from({ length: 100 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })); + pq.handleOverflow(events); + + // Allow async flush to complete + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + }); + + it('does not flush to disk below batch threshold', () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.handleOverflow([{ type: 'track', event: 'e1' }]); + + expect(mock.writeOverflowSnapshot).not.toHaveBeenCalled(); + }); + }); + + describe('flushOverflowBufferToDisk', () => { + it('writes buffer contents to overflow disk store', async () => { + const { store, mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.handleOverflow([ + { type: 'track', event: 'o1' }, + { type: 'track', event: 'o2' }, + ]); + + await pq.flushOverflowBufferToDisk(); + + expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + const written = JSON.parse(store.overflow!); + expect(written.version).toBe(1); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('o1'); + + // Buffer should be emptied + expect(pq.overflowBufferCount).toBe(0); + }); + + it('merges with existing overflow on disk', async () => { + const { store } = mockNativeStorage(); + + // Pre-populate overflow on disk + store.overflow = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'existing1' }], + }); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.handleOverflow([{ type: 'track', event: 'new1' }]); + await pq.flushOverflowBufferToDisk(); + + const written = JSON.parse(store.overflow!); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('existing1'); + expect(written.events[1].event).toBe('new1'); + }); + + it('enforces disk cap when merging', async () => { + const { store } = mockNativeStorage(); + + // Pre-populate with 4 events on disk + store.overflow = JSON.stringify({ + version: 1, + events: Array.from({ length: 4 }, (_, i) => ({ + type: 'track', + event: `old${i}`, + })), + }); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher, { + maxOfflineDiskEvents: 5, + }); + + pq.handleOverflow([ + { type: 'track', event: 'new1' }, + { type: 'track', event: 'new2' }, + ]); + await pq.flushOverflowBufferToDisk(); + + const written = JSON.parse(store.overflow!); + // 4 + 2 = 6, cap = 5, drop 1 oldest + expect(written.events.length).toBe(5); + // oldest (old0) should be dropped + expect(written.events[0].event).toBe('old1'); + }); + + it('is a no-op when buffer is empty', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.flushOverflowBufferToDisk(); + + expect(mock.writeOverflowSnapshot).not.toHaveBeenCalled(); + }); + }); + + describe('drainDiskToNetwork', () => { + it('sends overflow events directly to network without entering memory queue', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: [ + { type: 'track', event: 'overflow1' }, + { type: 'track', event: 'overflow2' }, + ], + }); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // Events sent via sendBatchDirect (fetchWithTimeout) + expect(dispatcher.getQueueRef().length).toBe(0); // memory queue untouched + expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalled(); + }); + + it('deletes overflow file after drain', async () => { + const { store, mock } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); + expect(store.overflow).toBeNull(); + }); + + it('stops on network failure', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: Array.from({ length: 5 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); + + const fetchMock = jest.fn(async () => ({ + ok: false, + status: 500, + })); + const dispatcher = createDispatcher({ + fetchWithTimeout: fetchMock, + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // Events should still be on disk + expect(store.overflow).not.toBeNull(); + const remaining = JSON.parse(store.overflow!); + expect(remaining.events.length).toBe(5); + }); + + it('drains in batches of 100', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: Array.from({ length: 250 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); + + const fetchCalls: number[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + fetchCalls.push(JSON.parse(init.body).batch.length); + return { ok: true, status: 200 } as any; + }), + }); + + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // Should have sent 3 batches: 100 + 100 + 50 + expect(fetchCalls).toEqual([100, 100, 50]); + expect(store.overflow).toBeNull(); + }); + + it('is a no-op when no overflow file exists', async () => { + mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect((dispatcher as any).opts.fetchWithTimeout).not.toHaveBeenCalled(); + }); + + it('filters expired events during drain', async () => { + const { store: overflowStore } = mockNativeStorage(); + + const eightDaysAgo = new Date( + Date.now() - 8 * 24 * 60 * 60 * 1000 + ).toISOString(); + const oneDayAgo = new Date( + Date.now() - 1 * 24 * 60 * 60 * 1000 + ).toISOString(); + + overflowStore.overflow = JSON.stringify({ + version: 1, + events: [ + { type: 'track', event: 'expired', timestamp: eightDaysAgo }, + { type: 'track', event: 'fresh', timestamp: oneDayAgo }, + ], + }); + + let sentBatch: any[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + sentBatch = JSON.parse(init.body).batch; + return { ok: true, status: 200 } as any; + }), + }); + + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(sentBatch.length).toBe(1); + expect(sentBatch[0].event).toBe('fresh'); + }); + + it('flushes overflow buffer before draining', async () => { + mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + // Add events to buffer (below batch threshold — won't auto-flush) + pq.handleOverflow([ + { type: 'track', event: 'buffered1' }, + { type: 'track', event: 'buffered2' }, + ]); + + let sentBatch: any[] = []; + // Override fetch to capture sent events + (dispatcher as any).opts.fetchWithTimeout = jest.fn( + async (_url?: string, init?: any) => { + sentBatch.push(...JSON.parse(init.body).batch); + return { ok: true, status: 200 } as any; + } + ); + + await pq.drainDiskToNetwork(dispatcher); + + // Buffered events should have been flushed to disk and then drained + expect(sentBatch.length).toBe(2); + expect(sentBatch[0].event).toBe('buffered1'); + }); + }); + + describe('flushToDisk with overflow', () => { + it('flushes overflow buffer to disk alongside queue snapshot', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + dispatcher.enqueue({ type: 'track', event: 'mem1' } as any); + + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + // Add overflow events to buffer + pq.handleOverflow([{ type: 'track', event: 'overflow1' }]); + + await pq.flushToDisk(); + + // Both queue snapshot and overflow should be written + expect(mock.writeSnapshot).toHaveBeenCalled(); + expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + }); + }); + + describe('deleteSnapshot with overflow', () => { + it('deletes both queue and overflow snapshots', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.handleOverflow([{ type: 'track', event: 'o1' }]); + + await pq.deleteSnapshot(); + + expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); + expect(pq.overflowBufferCount).toBe(0); + }); + }); + describe('full round-trip', () => { it('enqueue → flushToDisk → rehydrate preserves events', async () => { mockNativeStorage(); diff --git a/src/analytics/persistence/__tests__/integration.test.ts b/src/analytics/persistence/__tests__/integration.test.ts index 124243c..7675509 100644 --- a/src/analytics/persistence/__tests__/integration.test.ts +++ b/src/analytics/persistence/__tests__/integration.test.ts @@ -2,7 +2,10 @@ import { NativeModules, AppState } from 'react-native'; import { _resetRehydrationGuard } from '../PersistentEventQueue'; function mockNativeStorage() { - const store: { data: string | null } = { data: null }; + const store: { data: string | null; overflow: string | null } = { + data: null, + overflow: null, + }; NativeModules.MetaRouterQueueStorage = { readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { @@ -11,6 +14,13 @@ function mockNativeStorage() { deleteSnapshot: jest.fn(async () => { store.data = null; }), + readOverflowSnapshot: jest.fn(async () => store.overflow), + writeOverflowSnapshot: jest.fn(async (data: string) => { + store.overflow = data; + }), + deleteOverflowSnapshot: jest.fn(async () => { + store.overflow = null; + }), }; return { store, mock: NativeModules.MetaRouterQueueStorage }; } @@ -143,4 +153,87 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { expect(mock.deleteSnapshot).toHaveBeenCalled(); }); + + it('overflow events written to disk when memory queue overflows', async () => { + const { mock } = mockNativeStorage(); + + // Network fails so events stay in queue + (global.fetch as jest.Mock).mockImplementation(() => + Promise.reject(new Error('Network unavailable')) + ); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + // Small queue so we can trigger overflow easily + maxQueueBytes: 500, + maxOfflineDiskEvents: 100, + }, + { networkMonitor: monitor } + ); + await client.init(); + + // Track many events to overflow memory queue + for (let i = 0; i < 50; i++) { + client.track(`event_${i}`, { idx: i }); + } + + // Simulate app going to background to flush overflow to disk + await handleAppStateChange!('background'); + + // Overflow should have been written to disk + expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + }); + + it('events enqueue successfully while offline (no errors, no HTTP attempts)', async () => { + mockNativeStorage(); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }, + { networkMonitor: monitor } + ); + await client.init(); + + // Track events while offline + client.track('offline_event_1'); + client.track('offline_event_2'); + + // Flush should not make any HTTP calls + await client.flush(); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it('reset deletes both queue and overflow snapshots', async () => { + const { mock } = mockNativeStorage(); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const client = new MetaRouterAnalyticsClient({ + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }); + await client.init(); + + await client.reset(); + + expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); + }); }); diff --git a/src/analytics/persistence/types.ts b/src/analytics/persistence/types.ts index 9476758..5437d9d 100644 --- a/src/analytics/persistence/types.ts +++ b/src/analytics/persistence/types.ts @@ -20,3 +20,9 @@ export const FLUSH_THRESHOLD_BYTES = 2 * 1024 * 1024; /** Flush-to-disk threshold: event count */ export const FLUSH_THRESHOLD_COUNT = 500; + +/** Default max events stored on disk during extended offline periods */ +export const DEFAULT_MAX_OFFLINE_DISK_EVENTS = 10_000; + +/** Overflow buffer batch threshold: flush buffer to disk at this count */ +export const OVERFLOW_BATCH_THRESHOLD = 100; From cb80a57aa99588fb393fbe73b87fd640c418ee7a Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Wed, 15 Apr 2026 19:00:13 -0600 Subject: [PATCH 02/10] fix: improving flush and removing duplicated in memory queue --- .../MetaRouterAnalyticsClient.test.ts | 62 +-- src/analytics/MetaRouterAnalyticsClient.ts | 10 +- src/analytics/dispatcher.test.ts | 216 +++++++--- src/analytics/dispatcher.ts | 89 +++-- .../persistence/PersistentEventQueue.ts | 228 +++++++---- .../__tests__/PersistentEventQueue.test.ts | 369 +++++++++++------- .../persistence/__tests__/integration.test.ts | 48 ++- src/analytics/persistence/types.ts | 3 - src/analytics/utils/responseCategory.ts | 23 ++ 9 files changed, 703 insertions(+), 345 deletions(-) create mode 100644 src/analytics/utils/responseCategory.ts diff --git a/src/analytics/MetaRouterAnalyticsClient.test.ts b/src/analytics/MetaRouterAnalyticsClient.test.ts index e42b4cc..9d0c21c 100644 --- a/src/analytics/MetaRouterAnalyticsClient.test.ts +++ b/src/analytics/MetaRouterAnalyticsClient.test.ts @@ -266,29 +266,30 @@ describe('MetaRouterAnalyticsClient', () => { (client as any).dispatcher && ((client as any).dispatcher.opts.autoFlushThreshold = 9999); + // Spy on overflow to verify events get flushed + const flushSpy = jest.spyOn( + (client as any).persistentQueue, + 'flushEventsToOverflowDisk' + ); + // Track one event to measure enriched size, then set byte cap for 10 events client.track('e00'); const enrichedSize = JSON.stringify((client as any).queue[0]).length; (client as any).dispatcher.opts.maxQueueBytes = enrichedSize * 10; - const firstId = (client as any).queue[0].messageId; - - // Add 15 more (padded names for uniform size) => total 16, cap ~10 => drop 6 oldest + // Add 15 more (padded names for uniform size) => total 16, cap ~10 => overflow triggered for (let i = 1; i <= 15; i++) { client.track(`e${String(i).padStart(2, '0')}`); } - // Queue should be capped - expect((client as any).queue.length).toBe(10); + // Queue should not exceed cap (entire queue flushed on overflow, only latest events remain) + expect((client as any).queue.length).toBeLessThanOrEqual(10); - // The very first event should have been dropped - expect( - (client as any).queue.find((e: any) => e.messageId === firstId) - ).toBeUndefined(); + // Overflow should have been triggered + expect(flushSpy).toHaveBeenCalled(); - // Ordering should be preserved for the tail: remaining should be e06..e15 + // The most recent event should always be present const names = (client as any).queue.map((e: any) => e.event); - expect(names[0]).toBe('e06'); expect(names[names.length - 1]).toBe('e15'); // No network calls needed for this test @@ -633,13 +634,22 @@ describe('MetaRouterAnalyticsClient', () => { expect(fetch).toHaveBeenCalled(); }); - it('offline -> online transition resets circuit breaker and triggers flush', async () => { + it('offline -> online transition resets circuit breaker and triggers flush + drain', async () => { const monitor = new StubNetworkMonitor('connected'); const client = new MetaRouterAnalyticsClient(opts, { networkMonitor: monitor, }); await client.init(); + // Spy on flushEventsToOverflowDisk and drainDiskToNetwork + const overflowSpy = jest.spyOn( + (client as any).persistentQueue, + 'flushEventsToOverflowDisk' + ); + const drainSpy = jest + .spyOn((client as any).persistentQueue, 'drainDiskToNetwork') + .mockResolvedValue(undefined); + // Go offline monitor.simulate('disconnected'); @@ -647,20 +657,16 @@ describe('MetaRouterAnalyticsClient', () => { client.track('Offline Event 1'); client.track('Offline Event 2'); - // Flush should not send (offline) + // Flush while offline — events should be moved to overflow storage await client.flush(); - // Events should still be in queue since network is unavailable - // (the dispatcher guard prevents HTTP calls) + expect(overflowSpy).toHaveBeenCalled(); - // Go online — should trigger flush - (global as any).fetch = jest - .fn() - .mockResolvedValue({ ok: true, status: 200 }); + // Go online — should trigger flush and drain monitor.simulate('connected'); - // Allow the async flush to complete + // Allow the async operations to complete await new Promise((resolve) => setTimeout(resolve, 50)); - expect(fetch).toHaveBeenCalled(); + expect(drainSpy).toHaveBeenCalled(); }); it('reset() cleans up network monitor', async () => { @@ -675,7 +681,7 @@ describe('MetaRouterAnalyticsClient', () => { expect(stopSpy).toHaveBeenCalled(); }); - it('overflow events redirect to persistent queue on memory cap exceeded', async () => { + it('overflow events flush to overflow disk on memory cap exceeded', async () => { const monitor = new StubNetworkMonitor('disconnected'); const client = new MetaRouterAnalyticsClient( { @@ -687,6 +693,12 @@ describe('MetaRouterAnalyticsClient', () => { ); await client.init(); + // Spy on flushEventsToOverflowDisk + const flushSpy = jest.spyOn( + (client as any).persistentQueue, + 'flushEventsToOverflowDisk' + ); + // Override to prevent auto-flush from interfering jest.spyOn(client as any, 'flush').mockResolvedValue(undefined); (client as any).dispatcher.opts.autoFlushThreshold = 9999; @@ -696,10 +708,8 @@ describe('MetaRouterAnalyticsClient', () => { client.track(`event_${i}`, { data: 'x'.repeat(20) }); } - // Check that overflow buffer has events - expect( - (client as any).persistentQueue.overflowBufferCount - ).toBeGreaterThan(0); + // Check that events were flushed to overflow disk + expect(flushSpy).toHaveBeenCalled(); }); it('offline -> online drains disk overflow', async () => { diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index 1f974b8..e2241f9 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -121,7 +121,15 @@ export class MetaRouterAnalyticsClient { log, warn, error, - onOverflow: (events) => this.persistentQueue.handleOverflow(events), + onCapacityOverflow: (events) => + this.persistentQueue.flushEventsToOverflowDisk(events), + onFlushToOfflineStorage: (events) => + this.persistentQueue.flushEventsToOverflowDisk(events), + onFlushComplete: () => { + if (this.networkStatus === 'connected') { + void this.persistentQueue.drainDiskToNetwork(this.dispatcher); + } + }, onScheduleFlushIn: (ms) => { (this as any).scheduleFlushIn(ms, { fromDispatcher: true }); }, diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 11b19b5..10ad016 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -48,17 +48,27 @@ describe('Dispatcher', () => { expect(fetchSpy).toHaveBeenCalled(); }); - it('respects maxQueueBytes by dropping oldest', () => { + it('respects maxQueueBytes by flushing entire queue to overflow on capacity', () => { const opts = baseOpts(); // Each event is 28 chars: {"type":"track","event":"a"} // Set cap to fit exactly 3 events (84 chars) opts.maxQueueBytes = 84; + opts.onCapacityOverflow = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); d.enqueue({ type: 'track', event: 'b' } as any); d.enqueue({ type: 'track', event: 'c' } as any); - d.enqueue({ type: 'track', event: 'd' } as any); // drops oldest - expect(d.getQueueRef().map((e: any) => e.event)).toEqual(['b', 'c', 'd']); + d.enqueue({ type: 'track', event: 'd' } as any); // triggers full flush + enqueue 'd' + // Queue should only contain the new event 'd' + expect(d.getQueueRef().map((e: any) => e.event)).toEqual(['d']); + // Overflow callback should have received 'a', 'b', 'c' + expect(opts.onCapacityOverflow).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ event: 'a' }), + expect.objectContaining({ event: 'b' }), + expect.objectContaining({ event: 'c' }), + ]) + ); }); it('batches in chunks up to maxBatchSize', async () => { @@ -157,26 +167,21 @@ describe('Dispatcher', () => { d.enqueue(makeEvent(i) as any); } - // Queue should be capped at 2000 - expect(d.getQueueRef().length).toBe(2000); - - // Verify oldest events were dropped and newest remain + // With the new flush-entire-queue overflow model, each time the cap is exceeded + // the entire queue is dropped (no onCapacityOverflow set). + // Only the most recent event(s) that fit within the cap after the last overflow remain. const queue = d.getQueueRef(); - expect((queue[0] as any).properties.id).toBe('08000'); + // Queue should not exceed cap + expect(d.getQueueSizeBytes()).toBeLessThanOrEqual(eventSize * 2000); + // The last event should always be present expect((queue[queue.length - 1] as any).properties.id).toBe('09999'); - // Verify all IDs in queue are contiguous - for (let i = 0; i < 2000; i++) { - expect((queue[i] as any).properties.id).toBe( - String(8000 + i).padStart(5, '0') - ); - } - - // Verify warn was called 8000 times (once per dropped event) - expect((opts.warn as jest.Mock).mock.calls.length).toBe(8000); - expect((opts.warn as jest.Mock).mock.calls[0][0]).toContain( - 'Queue cap reached' - ); + // Verify warn was called for each overflow (entire queue dropped each time) + expect( + (opts.warn as jest.Mock).mock.calls.some((c) => + String(c[0]).includes('Queue cap reached') + ) + ).toBe(true); }); it('gradually recovers maxBatchSize after 413-induced reduction on subsequent successes', async () => { @@ -248,11 +253,12 @@ describe('Dispatcher', () => { ]); }); - it('enqueueFront respects maxQueueBytes by dropping oldest from front-loaded batch', () => { + it('enqueueFront flushes entire queue to overflow when exceeding capacity', () => { const opts = baseOpts(); // Each event is 28 chars — set cap to fit exactly 3 (84 chars) opts.maxQueueBytes = 84; opts.autoFlushThreshold = 9999; + opts.onCapacityOverflow = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); d.enqueue({ type: 'track', event: 'b' } as any); @@ -263,10 +269,14 @@ describe('Dispatcher', () => { { type: 'track', event: 'z' } as any, ]); - // Total would be 5×28=140 chars, cap is 84. After prepend: [x, y, z, a, b] -> drop oldest 2 -> [z, a, b] - const queue = d.getQueueRef(); - expect(queue.length).toBe(3); - expect(queue.map((e: any) => e.event)).toEqual(['z', 'a', 'b']); + // Total would be 5×28=140 chars, cap is 84. Entire merged queue flushed to overflow. + expect(opts.onCapacityOverflow).toHaveBeenCalledTimes(1); + const flushed = (opts.onCapacityOverflow as jest.Mock).mock.calls[0][0]; + expect(flushed.length).toBe(5); + expect(flushed.map((e: any) => e.event)).toEqual(['x', 'y', 'z', 'a', 'b']); + + // Queue should be empty after overflow flush + expect(d.getQueueRef().length).toBe(0); }); it('getQueueSizeBytes returns approximate serialized size', () => { @@ -408,7 +418,27 @@ describe('Dispatcher', () => { expect(d.getDebugInfo().isNetworkAvailable).toBe(false); }); - it('offline log warns with queue count', async () => { + it('offline log warns with flushed event count when onFlushToOfflineStorage is set', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.onFlushToOfflineStorage = jest.fn(); + const d = new Dispatcher(opts); + + d.enqueue({ type: 'track', event: 'e1' } as any); + d.enqueue({ type: 'track', event: 'e2' } as any); + await d.flush(); + + expect( + (opts.warn as jest.Mock).mock.calls.some( + (c) => + String(c[0]).includes('Offline') && + String(c[0]).includes('flushed') && + String(c[0]).includes('2 event(s)') + ) + ).toBe(true); + }); + + it('offline log warns with queue count when no onFlushToOfflineStorage', async () => { const opts = baseOpts(); opts.isNetworkAvailable = () => false; const d = new Dispatcher(opts); @@ -427,11 +457,11 @@ describe('Dispatcher', () => { }); }); - describe('onOverflow callback', () => { - it('fires with events evicted from queue instead of dropping', () => { + describe('onCapacityOverflow callback', () => { + it('fires with entire queue when capacity is hit', () => { const opts = baseOpts(); const overflowEvents: any[][] = []; - opts.onOverflow = jest.fn((events: any[]) => { + opts.onCapacityOverflow = jest.fn((events: any[]) => { overflowEvents.push(events); }); opts.maxQueueBytes = 84; // fits ~3 small events @@ -441,68 +471,136 @@ describe('Dispatcher', () => { d.enqueue({ type: 'track', event: 'a' } as any); d.enqueue({ type: 'track', event: 'b' } as any); d.enqueue({ type: 'track', event: 'c' } as any); - d.enqueue({ type: 'track', event: 'd' } as any); // evicts oldest - - expect(opts.onOverflow).toHaveBeenCalled(); - expect(overflowEvents[0].length).toBeGreaterThanOrEqual(1); - expect(overflowEvents[0][0].event).toBe('a'); + d.enqueue({ type: 'track', event: 'd' } as any); // flushes a,b,c to overflow + + expect(opts.onCapacityOverflow).toHaveBeenCalledTimes(1); + expect(overflowEvents[0].length).toBe(3); + expect(overflowEvents[0].map((e: any) => e.event)).toEqual([ + 'a', + 'b', + 'c', + ]); - // warn should NOT be called with "dropped oldest" when onOverflow is set - expect( - (opts.warn as jest.Mock).mock.calls.some((c) => - String(c[0]).includes('Queue cap reached') - ) - ).toBe(false); + // Queue should only contain the new event + expect(d.getQueueRef().length).toBe(1); + expect((d.getQueueRef()[0] as any).event).toBe('d'); }); it('does not fire when queue is within capacity', () => { const opts = baseOpts(); - opts.onOverflow = jest.fn(); + opts.onCapacityOverflow = jest.fn(); opts.autoFlushThreshold = 9999; const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); d.enqueue({ type: 'track', event: 'b' } as any); - expect(opts.onOverflow).not.toHaveBeenCalled(); + expect(opts.onCapacityOverflow).not.toHaveBeenCalled(); }); - it('batches multiple evictions into a single callback', () => { + it('resets queue bytes to 0 after overflow', () => { const opts = baseOpts(); - const overflowBatches: any[][] = []; - opts.onOverflow = jest.fn((events: any[]) => { - overflowBatches.push([...events]); - }); - // Set capacity to fit exactly 1 event + opts.onCapacityOverflow = jest.fn(); const eventSize = new TextEncoder().encode( JSON.stringify({ type: 'track', event: 'x' }) ).byteLength; - opts.maxQueueBytes = eventSize; + opts.maxQueueBytes = eventSize; // fits exactly 1 event opts.autoFlushThreshold = 9999; const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); - d.enqueue({ type: 'track', event: 'b' } as any); // evicts 'a' - d.enqueue({ type: 'track', event: 'c' } as any); // evicts 'b' + d.enqueue({ type: 'track', event: 'b' } as any); // overflows 'a', enqueues 'b' + + // Queue should have just 'b', size should be 1 event's worth + expect(d.getQueueRef().length).toBe(1); + expect(d.getQueueSizeBytes()).toBe(eventSize); + }); + }); + + describe('onFlushToOfflineStorage callback', () => { + it('fires when flush is called while offline', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.onFlushToOfflineStorage = jest.fn(); + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + await d.flush(); + + expect(opts.onFlushToOfflineStorage).toHaveBeenCalledTimes(1); + expect(opts.onFlushToOfflineStorage).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ event: 'a' }), + expect.objectContaining({ event: 'b' }), + ]) + ); + // Queue should be empty after flushing to offline storage + expect(d.getQueueRef().length).toBe(0); + }); + + it('does not fire when online', async () => { + const opts = baseOpts(); + opts.onFlushToOfflineStorage = jest.fn(); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + await d.flush(); + + expect(opts.onFlushToOfflineStorage).not.toHaveBeenCalled(); + }); + }); + + describe('onFlushComplete callback', () => { + it('fires after successful online flush that empties the queue', async () => { + const opts = baseOpts(); + opts.onFlushComplete = jest.fn(); + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + + await d.flush(); + + expect(opts.onFlushComplete).toHaveBeenCalledTimes(1); + }); + + it('does not fire on failed flush', async () => { + const opts = baseOpts(); + opts.onFlushComplete = jest.fn(); + opts.autoFlushThreshold = 9999; + opts.fetchWithTimeout = jest.fn(async () => ({ + ok: false, + status: 500, + headers: { get: () => null }, + })) as any; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + + await d.flush(); - // Each enqueue that causes eviction fires a callback - expect(opts.onOverflow).toHaveBeenCalledTimes(2); + expect(opts.onFlushComplete).not.toHaveBeenCalled(); }); }); describe('sendBatchDirect', () => { - it('returns true on successful send', async () => { + it('returns statusCode on successful send', async () => { const opts = baseOpts(); const d = new Dispatcher(opts); const result = await d.sendBatchDirect([ { type: 'track', event: 'e1' } as any, ]); - expect(result).toBe(true); + expect(result).toEqual({ statusCode: 200 }); expect(opts.fetchWithTimeout).toHaveBeenCalled(); }); - it('returns false on HTTP error', async () => { + it('returns statusCode on HTTP error', async () => { const opts = baseOpts(); opts.fetchWithTimeout = jest.fn(async () => ({ ok: false, @@ -513,10 +611,10 @@ describe('Dispatcher', () => { const result = await d.sendBatchDirect([ { type: 'track', event: 'e1' } as any, ]); - expect(result).toBe(false); + expect(result).toEqual({ statusCode: 500 }); }); - it('returns false on network error', async () => { + it('returns null on network error', async () => { const opts = baseOpts(); opts.fetchWithTimeout = jest.fn(async () => { throw new Error('Network unavailable'); @@ -526,7 +624,7 @@ describe('Dispatcher', () => { const result = await d.sendBatchDirect([ { type: 'track', event: 'e1' } as any, ]); - expect(result).toBe(false); + expect(result).toBeNull(); }); it('does not affect the memory queue', async () => { diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index e71edff..b7ef5ab 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -3,6 +3,10 @@ import type CircuitBreaker from './utils/circuitBreaker'; export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN'; +export interface NetworkResponse { + statusCode: number; +} + export interface DispatcherOptions { maxQueueBytes: number; // byte-based cap on memory queue (~5MB default) autoFlushThreshold: number; // e.g., 20 @@ -28,7 +32,9 @@ export interface DispatcherOptions { warn: (...args: any[]) => void; error: (...args: any[]) => void; - onOverflow?: (events: EventPayload[]) => void; // called with events evicted from memory queue when cap is hit + onCapacityOverflow?: (events: EventPayload[]) => void; // called with entire queue when byte cap is hit — flushes to overflow disk + onFlushToOfflineStorage?: (events: EventPayload[]) => void; // called when flush triggers while offline — persists queue to overflow disk + onFlushComplete?: () => void; // fires after successful online flush to trigger overflow drain onScheduleFlushIn?: (ms: number) => void; // optional notification for tests/metrics onFatalConfig?: () => void; // 401/403/404 handler } @@ -130,25 +136,23 @@ export default class Dispatcher { enqueue(event: EventPayload): void { const eventSize = this.estimateEventSize(event); - // Hard cap: evict oldest until there's room (byte limit) - const overflowBatch: EventPayload[] = []; - while ( + // Capacity overflow: flush entire queue to overflow disk, then reset + if ( this.queue.length > 0 && this.queueSizeBytes + eventSize > this.opts.maxQueueBytes ) { - const dropped = this.queue.shift(); - if (dropped) { - this.queueSizeBytes -= this.estimateEventSize(dropped); - if (this.opts.onOverflow) { - overflowBatch.push(dropped); - } else { - this.opts.warn('Queue cap reached — dropped oldest event'); - } + if (this.opts.onCapacityOverflow) { + const flushed = this.queue.splice(0); + this.queueSizeBytes = 0; + this.opts.onCapacityOverflow(flushed); + } else { + this.opts.warn( + `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` + ); + this.queue.length = 0; + this.queueSizeBytes = 0; } } - if (overflowBatch.length > 0) { - this.opts.onOverflow!(overflowBatch); - } this.opts.log( `Enqueuing event {"messageId": "${(event as any)?.messageId}", "type": "${(event as any)?.type}"}` @@ -173,14 +177,19 @@ export default class Dispatcher { this.queueSizeBytes += this.estimateEventSize(e); } - // Enforce cap: drop oldest (from front) until within byte limit - while ( - this.queue.length > 0 && - this.queueSizeBytes > this.opts.maxQueueBytes - ) { - const dropped = this.queue.shift(); - if (dropped) this.queueSizeBytes -= this.estimateEventSize(dropped); - this.opts.warn('Queue cap reached — dropped oldest event'); + // Enforce cap: if over byte limit, flush entire queue to overflow disk + if (this.queueSizeBytes > this.opts.maxQueueBytes) { + if (this.opts.onCapacityOverflow) { + const flushed = this.queue.splice(0); + this.queueSizeBytes = 0; + this.opts.onCapacityOverflow(flushed); + } else { + this.opts.warn( + `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` + ); + this.queue.length = 0; + this.queueSizeBytes = 0; + } } if (this.queue.length >= this.opts.autoFlushThreshold) { @@ -231,9 +240,18 @@ export default class Dispatcher { const doFlush = async () => { while (this.queue.length) { if (!this.opts.isNetworkAvailable()) { - this.opts.warn( - `Offline — pausing HTTP attempts, ${this.queue.length} event(s) queued` - ); + if (this.opts.onFlushToOfflineStorage) { + const flushed = this.queue.splice(0); + this.queueSizeBytes = 0; + this.opts.onFlushToOfflineStorage(flushed); + this.opts.warn( + `Offline — flushed ${flushed.length} event(s) to offline storage` + ); + } else { + this.opts.warn( + `Offline — pausing HTTP attempts, ${this.queue.length} event(s) queued` + ); + } return; } @@ -370,6 +388,11 @@ export default class Dispatcher { ); } this.opts.log('API call successful'); + + // If queue is now empty after this successful batch, fire onFlushComplete + if (this.queue.length === 0 && this.opts.isNetworkAvailable()) { + this.opts.onFlushComplete?.(); + } } catch (err) { this.circuit.onFailure(); this.consecutiveRetries += 1; @@ -408,9 +431,11 @@ export default class Dispatcher { /** * Send a batch directly to network, bypassing the memory queue. * Used by disk drain to flush overflow without loading into queue. - * Returns true on success, false on failure. + * Returns { statusCode } on HTTP response, null on network/transport error. */ - async sendBatchDirect(events: EventPayload[]): Promise { + async sendBatchDirect( + events: EventPayload[] + ): Promise { try { const nowIso = new Date().toISOString(); const batch = events.map((e) => ({ ...(e as any), sentAt: nowIso })); @@ -434,14 +459,14 @@ export default class Dispatcher { if (res.ok) { this.opts.log(`Direct batch send successful (${events.length} events)`); - return true; + } else { + this.opts.warn(`Direct batch send failed with status ${res.status}`); } - this.opts.warn(`Direct batch send failed with status ${res.status}`); - return false; + return { statusCode: res.status }; } catch (err) { this.opts.warn(`Direct batch send failed: ${(err as any)?.message}`); - return false; + return null; } } diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index 3efb71b..de83bba 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -14,9 +14,12 @@ import { FLUSH_THRESHOLD_COUNT, DEFAULT_EVENT_TTL_MS, DEFAULT_MAX_OFFLINE_DISK_EVENTS, - OVERFLOW_BATCH_THRESHOLD, type QueueSnapshot, } from './types'; +import { + ResponseCategory, + categorizeResponse, +} from '../utils/responseCategory'; import { log, warn } from '../utils/logger'; /** @@ -26,12 +29,17 @@ import { log, warn } from '../utils/logger'; */ let hasRehydrated = false; +/** + * Default batch size for draining overflow disk to network. + * May be halved on 413 responses. + */ +const DRAIN_BATCH_SIZE = 100; + export class PersistentEventQueue { private readonly dispatcher: Dispatcher; private flushInFlight: Promise | null = null; private _rehydratedEvents: number = 0; private readonly maxOfflineDiskEvents: number; - private overflowBuffer: EventPayload[] = []; private overflowFlushInFlight: Promise | null = null; private drainInFlight: Promise | null = null; @@ -151,11 +159,6 @@ export class PersistentEventQueue { await writeSnapshot(data); log(`Queue snapshot written to disk (${queue.length} events)`); } - - // Also flush overflow buffer to disk on background - if (this.overflowBuffer.length > 0) { - await this.flushOverflowBufferToDisk(); - } } catch (err) { warn('Failed to flush queue to disk:', err); } @@ -172,42 +175,35 @@ export class PersistentEventQueue { } /** - * Handle events evicted from the memory queue (overflow callback). - * Buffers events and flushes to disk in batches. + * Flush events to the overflow disk store. + * Called when the memory queue hits capacity (onCapacityOverflow) or + * when a flush triggers while offline (onFlushToOfflineStorage). + * The entire queue is drained and appended to the overflow disk file. + * Singleflight: concurrent calls coalesce into one write. */ - handleOverflow(events: EventPayload[]): void { - this.overflowBuffer.push(...events); - - // Enforce disk cap on buffer: drop oldest if overflow exceeds max - while (this.overflowBuffer.length > this.maxOfflineDiskEvents) { - this.overflowBuffer.shift(); - warn('Offline disk cap reached — dropped oldest overflow event'); - } - - if (this.overflowBuffer.length >= OVERFLOW_BATCH_THRESHOLD) { - void this.flushOverflowBufferToDisk(); - } + flushEventsToOverflowDisk(events: EventPayload[]): void { + if (events.length === 0) return; + void this._flushEventsToOverflowDisk(events); } - /** - * Flush the in-memory overflow buffer to the overflow disk store. - * Singleflight: concurrent calls coalesce into one write. - */ - async flushOverflowBufferToDisk(): Promise { - if (this.overflowFlushInFlight) return this.overflowFlushInFlight; + private async _flushEventsToOverflowDisk( + events: EventPayload[] + ): Promise { + // Singleflight: if a write is already in flight, queue up after it + if (this.overflowFlushInFlight) { + await this.overflowFlushInFlight; + } - this.overflowFlushInFlight = this._doFlushOverflowToDisk().finally(() => { - this.overflowFlushInFlight = null; - }); + this.overflowFlushInFlight = this._doWriteToOverflowDisk(events).finally( + () => { + this.overflowFlushInFlight = null; + } + ); return this.overflowFlushInFlight; } - private async _doFlushOverflowToDisk(): Promise { - if (this.overflowBuffer.length === 0) return; - + private async _doWriteToOverflowDisk(events: EventPayload[]): Promise { try { - const batch = this.overflowBuffer.splice(0); - // Read existing overflow on disk const raw = await readOverflowSnapshot(); let existing: EventPayload[] = []; @@ -225,7 +221,7 @@ export class PersistentEventQueue { } } - let combined = existing.concat(batch); + let combined = existing.concat(events); // Enforce disk cap if (combined.length > this.maxOfflineDiskEvents) { @@ -242,17 +238,24 @@ export class PersistentEventQueue { }; await writeOverflowSnapshot(JSON.stringify(snapshot)); log( - `Overflow buffer flushed to disk (${batch.length} new, ${combined.length} total on disk)` + `Flushed ${events.length} events to overflow disk (${combined.length} total on disk)` ); } catch (err) { - warn('Failed to flush overflow buffer to disk:', err); + warn('Failed to flush events to overflow disk:', err); } } /** * Drain overflow events directly from disk to network in batches. * Does NOT load events into the memory queue. - * Called on offline→online transition. + * Called on offline→online transition and after successful online flushes. + * + * Response handling: + * - 413: halves batch size, retries. Drops at batchSize=1 + * - 5xx/408/429: stops, retries on next online transition + * - 401/403/404: fatal, deletes overflow store + * - Other 4xx: drops batch, continues + * - Batch size restores after success */ async drainDiskToNetwork(dispatcher: Dispatcher): Promise { if (this.drainInFlight) return this.drainInFlight; @@ -264,10 +267,9 @@ export class PersistentEventQueue { } private async _doDrainDiskToNetwork(dispatcher: Dispatcher): Promise { - try { - // First, flush any remaining buffer to disk - await this.flushOverflowBufferToDisk(); + let batchSize = DRAIN_BATCH_SIZE; + try { while (true) { const raw = await readOverflowSnapshot(); if (!raw) return; @@ -303,52 +305,136 @@ export class PersistentEventQueue { } // Take a batch from the front (oldest first) - const batchSize = Math.min(100, events.length); - const batch = events.slice(0, batchSize); - const remaining = events.slice(batchSize); + const n = Math.min(batchSize, events.length); + const batch = events.slice(0, n); + const remaining = events.slice(n); - const success = await dispatcher.sendBatchDirect(batch); - if (success) { - if (remaining.length === 0) { - await nativeDeleteOverflowSnapshot(); - log('Overflow disk drain complete'); - } else { - const updated: QueueSnapshot = { - version: SNAPSHOT_VERSION, - events: remaining, - }; - await writeOverflowSnapshot(JSON.stringify(updated)); - log( - `Overflow drain batch sent (${batch.length}), ${remaining.length} remaining on disk` - ); - } - } else { - // Network failed — stop draining, will retry on next online transition + const response = await dispatcher.sendBatchDirect(batch); + + // Network/transport error — stop, retry on next online transition + if (!response) { log( - `Overflow drain paused — network send failed, ${events.length} events remain on disk` + `Overflow drain paused — network error, ${events.length} events remain on disk` ); + // Write back with expired events filtered + if (events.length !== snapshot.events.length) { + await writeOverflowSnapshot( + JSON.stringify({ version: SNAPSHOT_VERSION, events }) + ); + } return; } + + const category = categorizeResponse(response.statusCode); + + switch (category) { + case ResponseCategory.SUCCESS: { + // Restore batch size after success + if (batchSize < DRAIN_BATCH_SIZE) { + batchSize = Math.min(batchSize * 2, DRAIN_BATCH_SIZE); + } + + if (remaining.length === 0) { + await nativeDeleteOverflowSnapshot(); + log('Overflow disk drain complete'); + } else { + const updated: QueueSnapshot = { + version: SNAPSHOT_VERSION, + events: remaining, + }; + await writeOverflowSnapshot(JSON.stringify(updated)); + log( + `Overflow drain batch sent (${batch.length}), ${remaining.length} remaining on disk` + ); + } + break; + } + + case ResponseCategory.PAYLOAD_TOO_LARGE: { + if (batchSize > 1) { + batchSize = Math.max(1, Math.floor(batchSize / 2)); + warn(`Overflow drain: 413 — halving batch size to ${batchSize}`); + // Write back filtered events before retrying with smaller batch + if (events.length !== snapshot.events.length) { + await writeOverflowSnapshot( + JSON.stringify({ version: SNAPSHOT_VERSION, events }) + ); + } + // Continue loop with smaller batch + } else { + // batchSize=1, drop the offending event + const ids = (batch as any[]) + .map((e) => (e as any).messageId) + .join(','); + warn( + `Overflow drain: dropping oversize event(s) at batchSize=1; messageIds=${ids}` + ); + if (remaining.length === 0) { + await nativeDeleteOverflowSnapshot(); + } else { + await writeOverflowSnapshot( + JSON.stringify({ + version: SNAPSHOT_VERSION, + events: remaining, + }) + ); + } + } + break; + } + + case ResponseCategory.SERVER_ERROR: + case ResponseCategory.RATE_LIMITED: { + warn( + `Overflow drain paused — ${response.statusCode}, ${events.length} events remain on disk` + ); + // Write back with expired events filtered + if (events.length !== snapshot.events.length) { + await writeOverflowSnapshot( + JSON.stringify({ version: SNAPSHOT_VERSION, events }) + ); + } + return; + } + + case ResponseCategory.FATAL_CONFIG: { + warn( + `Overflow drain: fatal config error ${response.statusCode} — deleting overflow store` + ); + await nativeDeleteOverflowSnapshot(); + return; + } + + case ResponseCategory.CLIENT_ERROR: { + warn( + `Overflow drain: dropping batch due to client error ${response.statusCode}` + ); + if (remaining.length === 0) { + await nativeDeleteOverflowSnapshot(); + } else { + await writeOverflowSnapshot( + JSON.stringify({ + version: SNAPSHOT_VERSION, + events: remaining, + }) + ); + } + // Continue draining next batch + break; + } + } } } catch (err) { warn('Failed to drain overflow from disk:', err); } } - /** - * Get the number of events in the overflow buffer (not yet flushed to disk). - */ - get overflowBufferCount(): number { - return this.overflowBuffer.length; - } - /** * Delete the disk snapshot (used during reset). */ async deleteSnapshot(): Promise { await nativeDeleteSnapshot(); await nativeDeleteOverflowSnapshot(); - this.overflowBuffer.length = 0; } } diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 106794b..9c17ead 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -104,7 +104,7 @@ describe('PersistentEventQueue', () => { expect(dispatcher.getQueueRef().length).toBe(1); }); - it('enforces capacity cap on rehydrated events (drops oldest)', async () => { + it('flushes entire queue to overflow when rehydrated events exceed capacity', async () => { const { store } = mockNativeStorage(); // Use uniform-size events so byte cap maps to exact count const events = Array.from({ length: 2500 }, (_, i) => ({ @@ -115,15 +115,21 @@ describe('PersistentEventQueue', () => { const eventSize = JSON.stringify(events[0]).length; store.data = JSON.stringify({ version: 1, events }); - const dispatcher = createDispatcher({ maxQueueBytes: eventSize * 2000 }); + const overflowEvents: any[] = []; + const dispatcher = createDispatcher({ + maxQueueBytes: eventSize * 2000, + onCapacityOverflow: (evts: any[]) => { + overflowEvents.push(...evts); + }, + }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); await pq.rehydrate(); - expect(dispatcher.getQueueRef().length).toBe(2000); - // Should keep newest: events 500-2499 - expect((dispatcher.getQueueRef()[0] as any).properties.id).toBe('00500'); + // All 2500 events should have been flushed to overflow (exceeds cap) + expect(dispatcher.getQueueRef().length).toBe(0); + expect(overflowEvents.length).toBe(2500); }); it('skips rehydration if snapshot has unknown version', async () => { @@ -335,7 +341,7 @@ describe('PersistentEventQueue', () => { }); describe('deleteSnapshot', () => { - it('delegates to native module', async () => { + it('delegates to native module for both snapshots', async () => { const { mock } = mockNativeStorage(); const dispatcher = createDispatcher(); @@ -345,6 +351,7 @@ describe('PersistentEventQueue', () => { await pq.deleteSnapshot(); expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); }); }); @@ -554,97 +561,26 @@ describe('PersistentEventQueue', () => { }); }); - describe('handleOverflow', () => { - it('accumulates events in overflow buffer', () => { - mockNativeStorage(); + describe('flushEventsToOverflowDisk', () => { + it('writes events to overflow disk store', async () => { + const { store } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - pq.handleOverflow([ + pq.flushEventsToOverflowDisk([ { type: 'track', event: 'o1' }, { type: 'track', event: 'o2' }, ]); - expect(pq.overflowBufferCount).toBe(2); - }); - - it('respects maxOfflineDiskEvents cap on buffer', () => { - mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher, { - maxOfflineDiskEvents: 5, - }); - - const events = Array.from({ length: 10 }, (_, i) => ({ - type: 'track', - event: `e${i}`, - })); - pq.handleOverflow(events); - - // Buffer should be capped at 5 (oldest dropped) - expect(pq.overflowBufferCount).toBe(5); - }); - - it('triggers flush to disk at batch threshold', async () => { - const { mock } = mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - // Overflow exactly 100 events (the OVERFLOW_BATCH_THRESHOLD) - const events = Array.from({ length: 100 }, (_, i) => ({ - type: 'track', - event: `e${i}`, - })); - pq.handleOverflow(events); - - // Allow async flush to complete + // Allow async write to complete await new Promise((resolve) => setTimeout(resolve, 50)); - expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); - }); - - it('does not flush to disk below batch threshold', () => { - const { mock } = mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - pq.handleOverflow([{ type: 'track', event: 'e1' }]); - - expect(mock.writeOverflowSnapshot).not.toHaveBeenCalled(); - }); - }); - - describe('flushOverflowBufferToDisk', () => { - it('writes buffer contents to overflow disk store', async () => { - const { store, mock } = mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - pq.handleOverflow([ - { type: 'track', event: 'o1' }, - { type: 'track', event: 'o2' }, - ]); - - await pq.flushOverflowBufferToDisk(); - - expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); const written = JSON.parse(store.overflow!); expect(written.version).toBe(1); expect(written.events.length).toBe(2); expect(written.events[0].event).toBe('o1'); - - // Buffer should be emptied - expect(pq.overflowBufferCount).toBe(0); }); it('merges with existing overflow on disk', async () => { @@ -660,8 +596,9 @@ describe('PersistentEventQueue', () => { const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - pq.handleOverflow([{ type: 'track', event: 'new1' }]); - await pq.flushOverflowBufferToDisk(); + pq.flushEventsToOverflowDisk([{ type: 'track', event: 'new1' }]); + + await new Promise((resolve) => setTimeout(resolve, 50)); const written = JSON.parse(store.overflow!); expect(written.events.length).toBe(2); @@ -687,11 +624,12 @@ describe('PersistentEventQueue', () => { maxOfflineDiskEvents: 5, }); - pq.handleOverflow([ + pq.flushEventsToOverflowDisk([ { type: 'track', event: 'new1' }, { type: 'track', event: 'new2' }, ]); - await pq.flushOverflowBufferToDisk(); + + await new Promise((resolve) => setTimeout(resolve, 50)); const written = JSON.parse(store.overflow!); // 4 + 2 = 6, cap = 5, drop 1 oldest @@ -700,14 +638,16 @@ describe('PersistentEventQueue', () => { expect(written.events[0].event).toBe('old1'); }); - it('is a no-op when buffer is empty', async () => { + it('is a no-op when events array is empty', async () => { const { mock } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.flushOverflowBufferToDisk(); + pq.flushEventsToOverflowDisk([]); + + await new Promise((resolve) => setTimeout(resolve, 50)); expect(mock.writeOverflowSnapshot).not.toHaveBeenCalled(); }); @@ -754,7 +694,7 @@ describe('PersistentEventQueue', () => { expect(store.overflow).toBeNull(); }); - it('stops on network failure', async () => { + it('stops on 5xx server error', async () => { const { store } = mockNativeStorage(); store.overflow = JSON.stringify({ @@ -783,6 +723,183 @@ describe('PersistentEventQueue', () => { expect(remaining.events.length).toBe(5); }); + it('stops on 429 rate limit', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); + + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 429, + })), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(store.overflow).not.toBeNull(); + }); + + it('halves batch size on 413 and retries', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: Array.from({ length: 200 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); + + const batchSizes: number[] = []; + let callCount = 0; + const fetchMock = jest.fn(async (_url?: string, init?: any) => { + callCount++; + const batchLen = JSON.parse(init.body).batch.length; + batchSizes.push(batchLen); + // First call: 413 at batch size 100 + if (callCount === 1) { + return { ok: false, status: 413 }; + } + return { ok: true, status: 200 }; + }); + + const dispatcher = createDispatcher({ fetchWithTimeout: fetchMock }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // Call 1: 100 (413) — halve to 50 + // Call 2: 50 (success) — batch restores to 100, 150 remaining + // Call 3: 100 (success) — 50 remaining + // Call 4: 50 (success) — done + expect(batchSizes).toEqual([100, 50, 100, 50]); + expect(store.overflow).toBeNull(); + }); + + it('drops events on 413 at batchSize=1', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: [ + { type: 'track', event: 'big_event', messageId: 'msg1' }, + { type: 'track', event: 'normal_event', messageId: 'msg2' }, + ], + }); + + // Always return 413 + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 413, + })), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // All events should be dropped (413 at batchSize=1 drops them) + expect(store.overflow).toBeNull(); + }); + + it('deletes overflow store on fatal config error (401/403/404)', async () => { + const { store, mock } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); + + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 403, + })), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); + expect(store.overflow).toBeNull(); + }); + + it('drops batch on other 4xx and continues draining', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: Array.from({ length: 150 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); + + let callCount = 0; + const fetchMock = jest.fn(async () => { + callCount++; + if (callCount === 1) { + // First batch: 400 error — drop it and continue + return { ok: false, status: 400 }; + } + // Second batch: success + return { ok: true, status: 200 }; + }); + + const dispatcher = createDispatcher({ fetchWithTimeout: fetchMock }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // First batch dropped (100 events), second batch sent (50 events) + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(store.overflow).toBeNull(); + }); + + it('restores batch size after success following 413', async () => { + const { store } = mockNativeStorage(); + + store.overflow = JSON.stringify({ + version: 1, + events: Array.from({ length: 300 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); + + const batchSizes: number[] = []; + let callCount = 0; + const fetchMock = jest.fn(async (_url?: string, init?: any) => { + callCount++; + const batchLen = JSON.parse(init.body).batch.length; + batchSizes.push(batchLen); + if (callCount === 1) return { ok: false, status: 413 }; + return { ok: true, status: 200 }; + }); + + const dispatcher = createDispatcher({ fetchWithTimeout: fetchMock }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + // First: 100 (413), then 50 (success, restore), then 100 (success), then 100 (success) + expect(batchSizes[0]).toBe(100); // 413 + expect(batchSizes[1]).toBe(50); // halved + // After success, batch size doubles back toward 100 + expect(batchSizes[2]).toBe(100); + }); + it('drains in batches of 100', async () => { const { store } = mockNativeStorage(); @@ -859,72 +976,26 @@ describe('PersistentEventQueue', () => { expect(sentBatch[0].event).toBe('fresh'); }); - it('flushes overflow buffer before draining', async () => { - mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - // Add events to buffer (below batch threshold — won't auto-flush) - pq.handleOverflow([ - { type: 'track', event: 'buffered1' }, - { type: 'track', event: 'buffered2' }, - ]); - - let sentBatch: any[] = []; - // Override fetch to capture sent events - (dispatcher as any).opts.fetchWithTimeout = jest.fn( - async (_url?: string, init?: any) => { - sentBatch.push(...JSON.parse(init.body).batch); - return { ok: true, status: 200 } as any; - } - ); - - await pq.drainDiskToNetwork(dispatcher); - - // Buffered events should have been flushed to disk and then drained - expect(sentBatch.length).toBe(2); - expect(sentBatch[0].event).toBe('buffered1'); - }); - }); - - describe('flushToDisk with overflow', () => { - it('flushes overflow buffer to disk alongside queue snapshot', async () => { - const { mock } = mockNativeStorage(); - - const dispatcher = createDispatcher(); - dispatcher.enqueue({ type: 'track', event: 'mem1' } as any); - - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - // Add overflow events to buffer - pq.handleOverflow([{ type: 'track', event: 'overflow1' }]); - - await pq.flushToDisk(); - - // Both queue snapshot and overflow should be written - expect(mock.writeSnapshot).toHaveBeenCalled(); - expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); - }); - }); + it('stops on network/transport error (null response)', async () => { + const { store } = mockNativeStorage(); - describe('deleteSnapshot with overflow', () => { - it('deletes both queue and overflow snapshots', async () => { - const { mock } = mockNativeStorage(); + store.overflow = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); - const dispatcher = createDispatcher(); + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => { + throw new Error('Network unavailable'); + }), + }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - pq.handleOverflow([{ type: 'track', event: 'o1' }]); - - await pq.deleteSnapshot(); + await pq.drainDiskToNetwork(dispatcher); - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); - expect(pq.overflowBufferCount).toBe(0); + // Events should still be on disk + expect(store.overflow).not.toBeNull(); }); }); diff --git a/src/analytics/persistence/__tests__/integration.test.ts b/src/analytics/persistence/__tests__/integration.test.ts index 7675509..bc3871a 100644 --- a/src/analytics/persistence/__tests__/integration.test.ts +++ b/src/analytics/persistence/__tests__/integration.test.ts @@ -154,7 +154,7 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { expect(mock.deleteSnapshot).toHaveBeenCalled(); }); - it('overflow events written to disk when memory queue overflows', async () => { + it('overflow events written to overflow disk when memory queue overflows', async () => { const { mock } = mockNativeStorage(); // Network fails so events stay in queue @@ -185,8 +185,8 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { client.track(`event_${i}`, { idx: i }); } - // Simulate app going to background to flush overflow to disk - await handleAppStateChange!('background'); + // Allow async overflow writes to complete + await new Promise((resolve) => setTimeout(resolve, 100)); // Overflow should have been written to disk expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); @@ -214,7 +214,7 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { client.track('offline_event_1'); client.track('offline_event_2'); - // Flush should not make any HTTP calls + // Flush should not make any HTTP calls (events go to offline storage) await client.flush(); expect(global.fetch).not.toHaveBeenCalled(); }); @@ -236,4 +236,44 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { expect(mock.deleteSnapshot).toHaveBeenCalled(); expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); }); + + it('offline flush sends events to overflow disk, online drains them', async () => { + const { mock } = mockNativeStorage(); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }, + { networkMonitor: monitor } + ); + await client.init(); + + // Track events while offline + client.track('offline_1'); + client.track('offline_2'); + + // Flush while offline — events should go to overflow disk + await client.flush(); + + // Allow async overflow writes to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + expect(global.fetch).not.toHaveBeenCalled(); + + // Go online — should drain overflow disk to network + monitor.simulate('connected'); + + // Allow drain to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(global.fetch).toHaveBeenCalled(); + }); }); diff --git a/src/analytics/persistence/types.ts b/src/analytics/persistence/types.ts index 5437d9d..31d5ea1 100644 --- a/src/analytics/persistence/types.ts +++ b/src/analytics/persistence/types.ts @@ -23,6 +23,3 @@ export const FLUSH_THRESHOLD_COUNT = 500; /** Default max events stored on disk during extended offline periods */ export const DEFAULT_MAX_OFFLINE_DISK_EVENTS = 10_000; - -/** Overflow buffer batch threshold: flush buffer to disk at this count */ -export const OVERFLOW_BATCH_THRESHOLD = 100; diff --git a/src/analytics/utils/responseCategory.ts b/src/analytics/utils/responseCategory.ts new file mode 100644 index 0000000..c78a9e0 --- /dev/null +++ b/src/analytics/utils/responseCategory.ts @@ -0,0 +1,23 @@ +/** + * Shared classification of HTTP status codes used by both + * the dispatcher and the overflow disk drain. + */ +export enum ResponseCategory { + SUCCESS = 'SUCCESS', + SERVER_ERROR = 'SERVER_ERROR', + RATE_LIMITED = 'RATE_LIMITED', + PAYLOAD_TOO_LARGE = 'PAYLOAD_TOO_LARGE', + FATAL_CONFIG = 'FATAL_CONFIG', + CLIENT_ERROR = 'CLIENT_ERROR', +} + +export function categorizeResponse(statusCode: number): ResponseCategory { + if (statusCode >= 200 && statusCode < 300) return ResponseCategory.SUCCESS; + if (statusCode >= 500 || statusCode === 408) + return ResponseCategory.SERVER_ERROR; + if (statusCode === 429) return ResponseCategory.RATE_LIMITED; + if (statusCode === 413) return ResponseCategory.PAYLOAD_TOO_LARGE; + if (statusCode === 401 || statusCode === 403 || statusCode === 404) + return ResponseCategory.FATAL_CONFIG; + return ResponseCategory.CLIENT_ERROR; +} From 9ee0d900a4b565132f1bcece14f6e88ebef5f374 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Thu, 16 Apr 2026 19:36:51 -0600 Subject: [PATCH 03/10] refactor: consolidate disk storage into single queue.v1.json Collapses the dual-file (snapshot + overflow-snapshot) JS abstraction into one file backed by the existing native module's queue.v1.json. The overflow native methods were never implemented in iOS/Android, so in production the old "offline overflow" path silently dropped events; consolidation fixes this latent bug by routing all persistence through the single native API that already exists. - NativeQueueStorage: drop read/write/deleteOverflowSnapshot; single read/write/deleteSnapshot interface - PersistentEventQueue.flushEventsToDisk (was flushEventsToOverflowDisk) uses read-merge-cap-write semantics so crash-safety and offline-overflow writes share one disk store - flushToDisk now drains the memory queue and appends to disk (was: overwrite snapshot with current queue contents), avoiding duplication when both memory and disk hold events - Dispatcher.onFlushToOfflineStorage renamed to onFlushToDisk; the store is no longer offline-specific - New Dispatcher.drainQueue() helper for the persistence layer to empty memory and reset byte counter atomically - DEFAULT_MAX_OFFLINE_DISK_EVENTS renamed to DEFAULT_MAX_DISK_EVENTS internally; public InitOptions surface (maxOfflineDiskEvents) stays until the API rename commit Native modules are unchanged (already target queue.v1.json and already do atomic writes). Public InitOptions surface is unchanged. --- .../MetaRouterAnalyticsClient.test.ts | 10 +- src/analytics/MetaRouterAnalyticsClient.ts | 8 +- src/analytics/dispatcher.test.ts | 20 +- src/analytics/dispatcher.ts | 23 +- .../persistence/NativeQueueStorage.ts | 52 +-- .../persistence/PersistentEventQueue.ts | 307 ++++++++---------- .../__tests__/PersistentEventQueue.test.ts | 249 ++++++-------- .../persistence/__tests__/integration.test.ts | 57 +--- src/analytics/persistence/types.ts | 4 +- 9 files changed, 302 insertions(+), 428 deletions(-) diff --git a/src/analytics/MetaRouterAnalyticsClient.test.ts b/src/analytics/MetaRouterAnalyticsClient.test.ts index 9d0c21c..5821bdd 100644 --- a/src/analytics/MetaRouterAnalyticsClient.test.ts +++ b/src/analytics/MetaRouterAnalyticsClient.test.ts @@ -269,7 +269,7 @@ describe('MetaRouterAnalyticsClient', () => { // Spy on overflow to verify events get flushed const flushSpy = jest.spyOn( (client as any).persistentQueue, - 'flushEventsToOverflowDisk' + 'flushEventsToDisk' ); // Track one event to measure enriched size, then set byte cap for 10 events @@ -641,10 +641,10 @@ describe('MetaRouterAnalyticsClient', () => { }); await client.init(); - // Spy on flushEventsToOverflowDisk and drainDiskToNetwork + // Spy on flushEventsToDisk and drainDiskToNetwork const overflowSpy = jest.spyOn( (client as any).persistentQueue, - 'flushEventsToOverflowDisk' + 'flushEventsToDisk' ); const drainSpy = jest .spyOn((client as any).persistentQueue, 'drainDiskToNetwork') @@ -693,10 +693,10 @@ describe('MetaRouterAnalyticsClient', () => { ); await client.init(); - // Spy on flushEventsToOverflowDisk + // Spy on flushEventsToDisk const flushSpy = jest.spyOn( (client as any).persistentQueue, - 'flushEventsToOverflowDisk' + 'flushEventsToDisk' ); // Override to prevent auto-flush from interfering diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index e2241f9..7177c75 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -122,9 +122,9 @@ export class MetaRouterAnalyticsClient { warn, error, onCapacityOverflow: (events) => - this.persistentQueue.flushEventsToOverflowDisk(events), - onFlushToOfflineStorage: (events) => - this.persistentQueue.flushEventsToOverflowDisk(events), + void this.persistentQueue.flushEventsToDisk(events), + onFlushToDisk: (events) => + void this.persistentQueue.flushEventsToDisk(events), onFlushComplete: () => { if (this.networkStatus === 'connected') { void this.persistentQueue.drainDiskToNetwork(this.dispatcher); @@ -140,7 +140,7 @@ export class MetaRouterAnalyticsClient { this.queue = this.dispatcher.getQueueRef(); this.persistentQueue = new PersistentEventQueue(this.dispatcher, { - maxOfflineDiskEvents: options.maxOfflineDiskEvents, + maxDiskEvents: options.maxOfflineDiskEvents, }); } diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 10ad016..1de5e8f 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -418,10 +418,10 @@ describe('Dispatcher', () => { expect(d.getDebugInfo().isNetworkAvailable).toBe(false); }); - it('offline log warns with flushed event count when onFlushToOfflineStorage is set', async () => { + it('offline log warns with flushed event count when onFlushToDisk is set', async () => { const opts = baseOpts(); opts.isNetworkAvailable = () => false; - opts.onFlushToOfflineStorage = jest.fn(); + opts.onFlushToDisk = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'e1' } as any); @@ -438,7 +438,7 @@ describe('Dispatcher', () => { ).toBe(true); }); - it('offline log warns with queue count when no onFlushToOfflineStorage', async () => { + it('offline log warns with queue count when no onFlushToDisk', async () => { const opts = baseOpts(); opts.isNetworkAvailable = () => false; const d = new Dispatcher(opts); @@ -517,11 +517,11 @@ describe('Dispatcher', () => { }); }); - describe('onFlushToOfflineStorage callback', () => { + describe('onFlushToDisk callback', () => { it('fires when flush is called while offline', async () => { const opts = baseOpts(); opts.isNetworkAvailable = () => false; - opts.onFlushToOfflineStorage = jest.fn(); + opts.onFlushToDisk = jest.fn(); opts.autoFlushThreshold = 9999; const d = new Dispatcher(opts); @@ -530,20 +530,20 @@ describe('Dispatcher', () => { await d.flush(); - expect(opts.onFlushToOfflineStorage).toHaveBeenCalledTimes(1); - expect(opts.onFlushToOfflineStorage).toHaveBeenCalledWith( + expect(opts.onFlushToDisk).toHaveBeenCalledTimes(1); + expect(opts.onFlushToDisk).toHaveBeenCalledWith( expect.arrayContaining([ expect.objectContaining({ event: 'a' }), expect.objectContaining({ event: 'b' }), ]) ); - // Queue should be empty after flushing to offline storage + // Queue should be empty after flushing to disk expect(d.getQueueRef().length).toBe(0); }); it('does not fire when online', async () => { const opts = baseOpts(); - opts.onFlushToOfflineStorage = jest.fn(); + opts.onFlushToDisk = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); @@ -551,7 +551,7 @@ describe('Dispatcher', () => { await d.flush(); - expect(opts.onFlushToOfflineStorage).not.toHaveBeenCalled(); + expect(opts.onFlushToDisk).not.toHaveBeenCalled(); }); }); diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index b7ef5ab..6adb9fd 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -32,9 +32,9 @@ export interface DispatcherOptions { warn: (...args: any[]) => void; error: (...args: any[]) => void; - onCapacityOverflow?: (events: EventPayload[]) => void; // called with entire queue when byte cap is hit — flushes to overflow disk - onFlushToOfflineStorage?: (events: EventPayload[]) => void; // called when flush triggers while offline — persists queue to overflow disk - onFlushComplete?: () => void; // fires after successful online flush to trigger overflow drain + onCapacityOverflow?: (events: EventPayload[]) => void; // called with entire queue when byte cap is hit — flushes to disk + onFlushToDisk?: (events: EventPayload[]) => void; // called when flush triggers while offline — persists queue to disk + onFlushComplete?: () => void; // fires after successful online flush to trigger disk drain onScheduleFlushIn?: (ms: number) => void; // optional notification for tests/metrics onFatalConfig?: () => void; // 401/403/404 handler } @@ -63,6 +63,17 @@ export default class Dispatcher { return this.queue; } + /** + * Drain the in-memory queue entirely. Returns all events and resets the + * byte counter. Used by PersistentEventQueue to persist memory to disk + * without leaving duplicates behind. + */ + drainQueue(): EventPayload[] { + const events = this.queue.splice(0); + this.queueSizeBytes = 0; + return events; + } + start(): void { if (this.flushTimer) clearInterval(this.flushTimer); this.flushTimer = setInterval( @@ -240,12 +251,12 @@ export default class Dispatcher { const doFlush = async () => { while (this.queue.length) { if (!this.opts.isNetworkAvailable()) { - if (this.opts.onFlushToOfflineStorage) { + if (this.opts.onFlushToDisk) { const flushed = this.queue.splice(0); this.queueSizeBytes = 0; - this.opts.onFlushToOfflineStorage(flushed); + this.opts.onFlushToDisk(flushed); this.opts.warn( - `Offline — flushed ${flushed.length} event(s) to offline storage` + `Offline — flushed ${flushed.length} event(s) to disk` ); } else { this.opts.warn( diff --git a/src/analytics/persistence/NativeQueueStorage.ts b/src/analytics/persistence/NativeQueueStorage.ts index 09db7dd..f757ba7 100644 --- a/src/analytics/persistence/NativeQueueStorage.ts +++ b/src/analytics/persistence/NativeQueueStorage.ts @@ -2,25 +2,20 @@ import { NativeModules } from 'react-native'; import { warn } from '../utils/logger'; /** - * Native bridge for queue snapshot persistence. + * Native bridge for the single queue disk file (queue.v1.json). * - * Expected native module contract: - * - readSnapshot(): Promise — returns file contents or null - * - writeSnapshot(data: string): Promise — overwrites file with data - * - deleteSnapshot(): Promise — deletes file if it exists + * Native module contract (iOS + Android): + * - readSnapshot(): Promise — full contents, or null if no file + * - writeSnapshot(data: string): Promise — atomic overwrite + * - deleteSnapshot(): Promise — delete if present * - * Overflow snapshot methods (for offline disk overflow): - * - readOverflowSnapshot(): Promise - * - writeOverflowSnapshot(data: string): Promise - * - deleteOverflowSnapshot(): Promise + * Append / merge / cap logic lives in JS (PersistentEventQueue) rather than + * native so the policy stays in one place and is easy to test. */ interface NativeQueueStorageModule { readSnapshot(): Promise; writeSnapshot(data: string): Promise; deleteSnapshot(): Promise; - readOverflowSnapshot?(): Promise; - writeOverflowSnapshot?(data: string): Promise; - deleteOverflowSnapshot?(): Promise; } function getModule(): NativeQueueStorageModule | null { @@ -66,36 +61,3 @@ export async function deleteSnapshot(): Promise { warn('Failed to delete queue snapshot from disk:', err); } } - -// --- Overflow snapshot (offline disk overflow) --- - -export async function readOverflowSnapshot(): Promise { - const mod = getModule(); - if (!mod?.readOverflowSnapshot) return null; - try { - return await mod.readOverflowSnapshot(); - } catch (err) { - warn('Failed to read overflow snapshot from disk:', err); - return null; - } -} - -export async function writeOverflowSnapshot(data: string): Promise { - const mod = getModule(); - if (!mod?.writeOverflowSnapshot) return; - try { - await mod.writeOverflowSnapshot(data); - } catch (err) { - warn('Failed to write overflow snapshot to disk:', err); - } -} - -export async function deleteOverflowSnapshot(): Promise { - const mod = getModule(); - if (!mod?.deleteOverflowSnapshot) return; - try { - await mod.deleteOverflowSnapshot(); - } catch (err) { - warn('Failed to delete overflow snapshot from disk:', err); - } -} diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index de83bba..34fdbdc 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -4,16 +4,13 @@ import { readSnapshot, writeSnapshot, deleteSnapshot as nativeDeleteSnapshot, - readOverflowSnapshot, - writeOverflowSnapshot, - deleteOverflowSnapshot as nativeDeleteOverflowSnapshot, } from './NativeQueueStorage'; import { SNAPSHOT_VERSION, FLUSH_THRESHOLD_BYTES, FLUSH_THRESHOLD_COUNT, DEFAULT_EVENT_TTL_MS, - DEFAULT_MAX_OFFLINE_DISK_EVENTS, + DEFAULT_MAX_DISK_EVENTS, type QueueSnapshot, } from './types'; import { @@ -30,26 +27,32 @@ import { log, warn } from '../utils/logger'; let hasRehydrated = false; /** - * Default batch size for draining overflow disk to network. + * Default batch size for draining disk events to network. * May be halved on 413 responses. */ const DRAIN_BATCH_SIZE = 100; +/** + * Memory + disk coordination for the event queue. + * + * Single disk file (queue.v1.json) backs both crash-safety (background flush) + * and offline overflow. All writes go through {@link flushEventsToDisk} which + * reads existing disk contents, merges the new events, enforces the disk cap, + * and atomically writes back. + */ export class PersistentEventQueue { private readonly dispatcher: Dispatcher; - private flushInFlight: Promise | null = null; + private readonly maxDiskEvents: number; private _rehydratedEvents: number = 0; - private readonly maxOfflineDiskEvents: number; - private overflowFlushInFlight: Promise | null = null; + + /** Serializes all disk writes (flushEventsToDisk, flushToDisk). */ + private diskWriteInFlight: Promise | null = null; + /** Guards drainDiskToNetwork (read-then-delete semantics). */ private drainInFlight: Promise | null = null; - constructor( - dispatcher: Dispatcher, - opts?: { maxOfflineDiskEvents?: number } - ) { + constructor(dispatcher: Dispatcher, opts?: { maxDiskEvents?: number }) { this.dispatcher = dispatcher; - this.maxOfflineDiskEvents = - opts?.maxOfflineDiskEvents ?? DEFAULT_MAX_OFFLINE_DISK_EVENTS; + this.maxDiskEvents = opts?.maxDiskEvents ?? DEFAULT_MAX_DISK_EVENTS; } get rehydratedEvents(): number { @@ -59,6 +62,9 @@ export class PersistentEventQueue { /** * Rehydrate events from disk into the in-memory queue. * Only runs once per process lifetime. + * + * (NOTE: C3 will replace this with a cheap exists() check + hasDiskData flag + * so disk events drain directly to network instead of being loaded into memory.) */ async rehydrate(): Promise { if (hasRehydrated) { @@ -97,16 +103,7 @@ export class PersistentEventQueue { return; } - // Filter out events older than TTL - const now = Date.now(); - const cutoff = now - DEFAULT_EVENT_TTL_MS; - const fresh = snapshot.events.filter((e) => { - const ts = (e as any).timestamp; - if (!ts) return true; // keep events without timestamp - const eventTime = new Date(ts).getTime(); - return !isNaN(eventTime) && eventTime > cutoff; - }); - + const fresh = filterExpired(snapshot.events); const expired = snapshot.events.length - fresh.length; if (expired > 0) { warn(`Dropped ${expired} expired events (older than 7 days)`); @@ -122,7 +119,6 @@ export class PersistentEventQueue { this._rehydratedEvents = fresh.length; log(`Rehydrated ${fresh.length} events from disk`); - // Clean up disk after successful rehydration await nativeDeleteSnapshot(); } catch (err) { warn('Failed to rehydrate queue from disk:', err); @@ -130,132 +126,91 @@ export class PersistentEventQueue { } /** - * Flush current in-memory queue state to disk. - * Serialized: concurrent calls coalesce into one write. + * Flush the in-memory queue to disk (append + cap). + * Called on app background and when the flush-to-disk threshold is hit. + * Serialized against other disk writes. */ async flushToDisk(): Promise { - if (this.flushInFlight) return this.flushInFlight; - - this.flushInFlight = this._doFlushToDisk().finally(() => { - this.flushInFlight = null; - }); - return this.flushInFlight; - } - - private async _doFlushToDisk(): Promise { - try { - const queue = this.dispatcher.getQueueRef(); - - if (queue.length === 0) { - log('Queue empty — deleting snapshot'); - await nativeDeleteSnapshot(); - } else { - const snapshot: QueueSnapshot = { - version: SNAPSHOT_VERSION, - events: [...queue], // shallow copy to avoid mutation during async write - }; - - const data = JSON.stringify(snapshot); - await writeSnapshot(data); - log(`Queue snapshot written to disk (${queue.length} events)`); - } - } catch (err) { - warn('Failed to flush queue to disk:', err); + const queue = this.dispatcher.getQueueRef(); + if (queue.length === 0) { + // Nothing in memory to persist. Leave any existing disk state alone. + return; } + const events = this.dispatcher.drainQueue(); + return this.flushEventsToDisk(events); } /** - * Check if the queue has crossed a flush-to-disk threshold. - */ - shouldFlushToDisk(): boolean { - return ( - this.dispatcher.getQueueRef().length >= FLUSH_THRESHOLD_COUNT || - this.dispatcher.getQueueSizeBytes() >= FLUSH_THRESHOLD_BYTES - ); - } - - /** - * Flush events to the overflow disk store. - * Called when the memory queue hits capacity (onCapacityOverflow) or - * when a flush triggers while offline (onFlushToOfflineStorage). - * The entire queue is drained and appended to the overflow disk file. - * Singleflight: concurrent calls coalesce into one write. + * Append explicit events to the disk store (read-merge-cap-write). + * Called by dispatcher onCapacityOverflow / onFlushToDisk callbacks. + * Serialized against other disk writes. */ - flushEventsToOverflowDisk(events: EventPayload[]): void { + async flushEventsToDisk(events: EventPayload[]): Promise { if (events.length === 0) return; - void this._flushEventsToOverflowDisk(events); - } - private async _flushEventsToOverflowDisk( - events: EventPayload[] - ): Promise { - // Singleflight: if a write is already in flight, queue up after it - if (this.overflowFlushInFlight) { - await this.overflowFlushInFlight; - } - - this.overflowFlushInFlight = this._doWriteToOverflowDisk(events).finally( - () => { - this.overflowFlushInFlight = null; - } - ); - return this.overflowFlushInFlight; + // Chain onto any in-flight write so writes are ordered. + const prev = this.diskWriteInFlight; + const next = (async () => { + if (prev) await prev; + await this._doFlushEventsToDisk(events); + })(); + this.diskWriteInFlight = next.finally(() => { + if (this.diskWriteInFlight === next) this.diskWriteInFlight = null; + }); + return this.diskWriteInFlight; } - private async _doWriteToOverflowDisk(events: EventPayload[]): Promise { + private async _doFlushEventsToDisk(events: EventPayload[]): Promise { try { - // Read existing overflow on disk - const raw = await readOverflowSnapshot(); - let existing: EventPayload[] = []; - if (raw) { - try { - const snapshot: QueueSnapshot = JSON.parse(raw); - if ( - snapshot.version === SNAPSHOT_VERSION && - Array.isArray(snapshot.events) - ) { - existing = snapshot.events; - } - } catch { - warn('Overflow snapshot is corrupt JSON — overwriting'); - } - } - + const existing = await this._readExistingEvents(); let combined = existing.concat(events); - // Enforce disk cap - if (combined.length > this.maxOfflineDiskEvents) { - const dropCount = combined.length - this.maxOfflineDiskEvents; + // Enforce disk cap by dropping oldest + if (combined.length > this.maxDiskEvents) { + const dropCount = combined.length - this.maxDiskEvents; combined = combined.slice(dropCount); - warn( - `Offline overflow disk cap enforced — dropped ${dropCount} oldest events` - ); + warn(`Disk store cap reached — dropped ${dropCount} oldest events`); + } + + if (combined.length === 0) { + await nativeDeleteSnapshot(); + return; } const snapshot: QueueSnapshot = { version: SNAPSHOT_VERSION, events: combined, }; - await writeOverflowSnapshot(JSON.stringify(snapshot)); + await writeSnapshot(JSON.stringify(snapshot)); log( - `Flushed ${events.length} events to overflow disk (${combined.length} total on disk)` + `Memory queue flushed to disk: ${events.length} events, ${combined.length} total on disk` ); } catch (err) { - warn('Failed to flush events to overflow disk:', err); + warn('Failed to flush events to disk:', err); } } /** - * Drain overflow events directly from disk to network in batches. + * Check if the queue has crossed a flush-to-disk threshold. + */ + shouldFlushToDisk(): boolean { + return ( + this.dispatcher.getQueueRef().length >= FLUSH_THRESHOLD_COUNT || + this.dispatcher.getQueueSizeBytes() >= FLUSH_THRESHOLD_BYTES + ); + } + + /** + * Drain disk events directly to network in batches. * Does NOT load events into the memory queue. * Called on offline→online transition and after successful online flushes. * * Response handling: + * - 200-299: advances and deletes sent events; restores batch size after 413 * - 413: halves batch size, retries. Drops at batchSize=1 - * - 5xx/408/429: stops, retries on next online transition - * - 401/403/404: fatal, deletes overflow store + * - 5xx/408/429: stops, writes remainder back, retries on next online transition + * - 401/403/404: fatal, deletes disk store * - Other 4xx: drops batch, continues - * - Batch size restores after success */ async drainDiskToNetwork(dispatcher: Dispatcher): Promise { if (this.drainInFlight) return this.drainInFlight; @@ -271,36 +226,34 @@ export class PersistentEventQueue { try { while (true) { - const raw = await readOverflowSnapshot(); + const raw = await readSnapshot(); if (!raw) return; let snapshot: QueueSnapshot; try { snapshot = JSON.parse(raw); } catch { - warn('Overflow snapshot is corrupt during drain — deleting'); - await nativeDeleteOverflowSnapshot(); + warn('Queue snapshot is corrupt during drain — deleting'); + await nativeDeleteSnapshot(); return; } if (!Array.isArray(snapshot.events) || snapshot.events.length === 0) { - await nativeDeleteOverflowSnapshot(); + await nativeDeleteSnapshot(); return; } - // Filter expired events - const now = Date.now(); - const cutoff = now - DEFAULT_EVENT_TTL_MS; - const events = snapshot.events.filter((e) => { - const ts = (e as any).timestamp; - if (!ts) return true; - const eventTime = new Date(ts).getTime(); - return !isNaN(eventTime) && eventTime > cutoff; - }); + const events = filterExpired(snapshot.events); + const droppedExpired = snapshot.events.length - events.length; + if (droppedExpired > 0) { + log( + `Drain TTL filter dropped ${droppedExpired} event(s) older than 7 days` + ); + } if (events.length === 0) { - log('All overflow events expired — deleting'); - await nativeDeleteOverflowSnapshot(); + log('All disk events expired — deleting'); + await nativeDeleteSnapshot(); return; } @@ -314,11 +267,10 @@ export class PersistentEventQueue { // Network/transport error — stop, retry on next online transition if (!response) { log( - `Overflow drain paused — network error, ${events.length} events remain on disk` + `Disk drain paused — network error, ${events.length} event(s) remain on disk` ); - // Write back with expired events filtered - if (events.length !== snapshot.events.length) { - await writeOverflowSnapshot( + if (droppedExpired > 0) { + await writeSnapshot( JSON.stringify({ version: SNAPSHOT_VERSION, events }) ); } @@ -329,22 +281,22 @@ export class PersistentEventQueue { switch (category) { case ResponseCategory.SUCCESS: { - // Restore batch size after success if (batchSize < DRAIN_BATCH_SIZE) { batchSize = Math.min(batchSize * 2, DRAIN_BATCH_SIZE); } if (remaining.length === 0) { - await nativeDeleteOverflowSnapshot(); - log('Overflow disk drain complete'); + await nativeDeleteSnapshot(); + log('Disk store drain complete'); } else { - const updated: QueueSnapshot = { - version: SNAPSHOT_VERSION, - events: remaining, - }; - await writeOverflowSnapshot(JSON.stringify(updated)); + await writeSnapshot( + JSON.stringify({ + version: SNAPSHOT_VERSION, + events: remaining, + }) + ); log( - `Overflow drain batch sent (${batch.length}), ${remaining.length} remaining on disk` + `Disk drain batch sent (${batch.length}), ${remaining.length} remaining on disk` ); } break; @@ -353,26 +305,24 @@ export class PersistentEventQueue { case ResponseCategory.PAYLOAD_TOO_LARGE: { if (batchSize > 1) { batchSize = Math.max(1, Math.floor(batchSize / 2)); - warn(`Overflow drain: 413 — halving batch size to ${batchSize}`); - // Write back filtered events before retrying with smaller batch - if (events.length !== snapshot.events.length) { - await writeOverflowSnapshot( + warn(`Disk drain: 413 — halving batch size to ${batchSize}`); + if (droppedExpired > 0) { + await writeSnapshot( JSON.stringify({ version: SNAPSHOT_VERSION, events }) ); } // Continue loop with smaller batch } else { - // batchSize=1, drop the offending event const ids = (batch as any[]) .map((e) => (e as any).messageId) .join(','); warn( - `Overflow drain: dropping oversize event(s) at batchSize=1; messageIds=${ids}` + `Disk drain: dropping oversize event(s) at batchSize=1; messageIds=${ids}` ); if (remaining.length === 0) { - await nativeDeleteOverflowSnapshot(); + await nativeDeleteSnapshot(); } else { - await writeOverflowSnapshot( + await writeSnapshot( JSON.stringify({ version: SNAPSHOT_VERSION, events: remaining, @@ -386,11 +336,10 @@ export class PersistentEventQueue { case ResponseCategory.SERVER_ERROR: case ResponseCategory.RATE_LIMITED: { warn( - `Overflow drain paused — ${response.statusCode}, ${events.length} events remain on disk` + `Disk drain paused — ${response.statusCode}, ${events.length} event(s) remain on disk` ); - // Write back with expired events filtered - if (events.length !== snapshot.events.length) { - await writeOverflowSnapshot( + if (droppedExpired > 0) { + await writeSnapshot( JSON.stringify({ version: SNAPSHOT_VERSION, events }) ); } @@ -399,43 +348,69 @@ export class PersistentEventQueue { case ResponseCategory.FATAL_CONFIG: { warn( - `Overflow drain: fatal config error ${response.statusCode} — deleting overflow store` + `Disk drain: fatal config error ${response.statusCode} — deleting disk store` ); - await nativeDeleteOverflowSnapshot(); + await nativeDeleteSnapshot(); return; } case ResponseCategory.CLIENT_ERROR: { warn( - `Overflow drain: dropping batch due to client error ${response.statusCode}` + `Disk drain: dropping batch due to client error ${response.statusCode}` ); if (remaining.length === 0) { - await nativeDeleteOverflowSnapshot(); + await nativeDeleteSnapshot(); } else { - await writeOverflowSnapshot( + await writeSnapshot( JSON.stringify({ version: SNAPSHOT_VERSION, events: remaining, }) ); } - // Continue draining next batch break; } } } } catch (err) { - warn('Failed to drain overflow from disk:', err); + warn('Failed to drain disk to network:', err); } } /** - * Delete the disk snapshot (used during reset). + * Delete the disk store (used during reset). */ async deleteSnapshot(): Promise { await nativeDeleteSnapshot(); - await nativeDeleteOverflowSnapshot(); } + + private async _readExistingEvents(): Promise { + const raw = await readSnapshot(); + if (!raw) return []; + try { + const snapshot: QueueSnapshot = JSON.parse(raw); + if ( + snapshot.version === SNAPSHOT_VERSION && + Array.isArray(snapshot.events) + ) { + return snapshot.events; + } + return []; + } catch { + warn('Existing disk snapshot is corrupt JSON — overwriting'); + return []; + } + } +} + +function filterExpired(events: EventPayload[]): EventPayload[] { + const cutoff = Date.now() - DEFAULT_EVENT_TTL_MS; + return events.filter((e) => { + const ts = (e as any).timestamp; + if (!ts) return true; + const eventTime = new Date(ts).getTime(); + return !isNaN(eventTime) && eventTime > cutoff; + }); } /** diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 9c17ead..369c686 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -2,12 +2,8 @@ import { NativeModules } from 'react-native'; import Dispatcher from '../../dispatcher'; import CircuitBreaker from '../../utils/circuitBreaker'; -// Helpers function mockNativeStorage() { - const store: { data: string | null; overflow: string | null } = { - data: null, - overflow: null, - }; + const store: { data: string | null } = { data: null }; NativeModules.MetaRouterQueueStorage = { readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { @@ -16,23 +12,18 @@ function mockNativeStorage() { deleteSnapshot: jest.fn(async () => { store.data = null; }), - readOverflowSnapshot: jest.fn(async () => store.overflow), - writeOverflowSnapshot: jest.fn(async (data: string) => { - store.overflow = data; - }), - deleteOverflowSnapshot: jest.fn(async () => { - store.overflow = null; - }), }; return { store, mock: NativeModules.MetaRouterQueueStorage }; } function createDispatcher(overrides?: Partial) { return new Dispatcher({ - maxQueueBytes: 5 * 1024 * 1024, // 5MB + maxQueueBytes: 5 * 1024 * 1024, autoFlushThreshold: 9999, maxBatchSize: 100, flushIntervalSeconds: 3600, + baseRetryDelayMs: 1000, + maxRetryDelayMs: 8000, isNetworkAvailable: () => true, endpoint: (p: string) => `https://example.com${p}`, fetchWithTimeout: jest.fn(async () => ({ ok: true, status: 200 }) as any), @@ -57,7 +48,6 @@ function createDispatcher(overrides?: Partial) { describe('PersistentEventQueue', () => { beforeEach(() => { jest.clearAllMocks(); - // Reset the rehydration guard before each test const { _resetRehydrationGuard } = require('../PersistentEventQueue'); _resetRehydrationGuard(); }); @@ -99,12 +89,11 @@ describe('PersistentEventQueue', () => { await pq.rehydrate(); await pq.rehydrate(); - // readSnapshot should only be called once expect(mock.readSnapshot).toHaveBeenCalledTimes(1); expect(dispatcher.getQueueRef().length).toBe(1); }); - it('flushes entire queue to overflow when rehydrated events exceed capacity', async () => { + it('flushes entire queue to disk when rehydrated events exceed memory byte cap', async () => { const { store } = mockNativeStorage(); // Use uniform-size events so byte cap maps to exact count const events = Array.from({ length: 2500 }, (_, i) => ({ @@ -127,7 +116,6 @@ describe('PersistentEventQueue', () => { await pq.rehydrate(); - // All 2500 events should have been flushed to overflow (exceeds cap) expect(dispatcher.getQueueRef().length).toBe(0); expect(overflowEvents.length).toBe(2500); }); @@ -239,7 +227,6 @@ describe('PersistentEventQueue', () => { await pq.rehydrate(); expect(dispatcher.getQueueRef().length).toBe(1); - // Reset guard and provide new snapshot _resetRehydrationGuard(); store.data = JSON.stringify({ version: 1, @@ -249,14 +236,13 @@ describe('PersistentEventQueue', () => { const pq2 = new PersistentEventQueue(dispatcher); await pq2.rehydrate(); - // Should have rehydrated the second time expect(mock.readSnapshot).toHaveBeenCalledTimes(2); }); }); describe('flushToDisk', () => { - it('writes current queue state to disk as versioned JSON', async () => { - const { mock } = mockNativeStorage(); + it('drains memory queue and appends to disk as versioned JSON', async () => { + const { mock, store } = mockNativeStorage(); const dispatcher = createDispatcher(); dispatcher.enqueue({ @@ -276,13 +262,15 @@ describe('PersistentEventQueue', () => { await pq.flushToDisk(); expect(mock.writeSnapshot).toHaveBeenCalledTimes(1); - const written = JSON.parse(mock.writeSnapshot.mock.calls[0][0]); + const written = JSON.parse(store.data!); expect(written.version).toBe(1); expect(written.events.length).toBe(2); expect(written.events[0].event).toBe('mem1'); + // Memory queue drained + expect(dispatcher.getQueueRef().length).toBe(0); }); - it('deletes snapshot if queue is empty', async () => { + it('is a no-op when memory queue is empty', async () => { const { mock } = mockNativeStorage(); const dispatcher = createDispatcher(); @@ -291,36 +279,30 @@ describe('PersistentEventQueue', () => { await pq.flushToDisk(); - expect(mock.deleteSnapshot).toHaveBeenCalled(); expect(mock.writeSnapshot).not.toHaveBeenCalled(); + expect(mock.deleteSnapshot).not.toHaveBeenCalled(); }); - it('serializes flushToDisk calls (no concurrent writes)', async () => { - const { mock } = mockNativeStorage(); + it('merges memory events with existing disk events', async () => { + const { store } = mockNativeStorage(); - let writeResolve: (() => void) | null = null; - mock.writeSnapshot = jest.fn( - () => - new Promise((resolve) => { - writeResolve = resolve; - }) - ); + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'disk1' }], + }); const dispatcher = createDispatcher(); - dispatcher.enqueue({ type: 'track', event: 'e1' } as any); + dispatcher.enqueue({ type: 'track', event: 'mem1' } as any); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - const p1 = pq.flushToDisk(); - const p2 = pq.flushToDisk(); - - // Only one write should be in flight - expect(mock.writeSnapshot).toHaveBeenCalledTimes(1); + await pq.flushToDisk(); - writeResolve!(); - await p1; - await p2; + const written = JSON.parse(store.data!); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('disk1'); + expect(written.events[1].event).toBe('mem1'); }); }); @@ -338,38 +320,19 @@ describe('PersistentEventQueue', () => { expect(pq.shouldFlushToDisk()).toBe(false); }); - }); - describe('deleteSnapshot', () => { - it('delegates to native module for both snapshots', async () => { - const { mock } = mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.deleteSnapshot(); - - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); - }); - }); - - describe('size-based flush threshold', () => { - it('shouldFlushToDisk returns true when byte size exceeds threshold', () => { + it('returns true when byte size exceeds threshold', () => { mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - // Create a large event (~2KB each, need ~1000 to reach 2MB) const bigProps: Record = {}; for (let i = 0; i < 20; i++) { bigProps[`key${i}`] = 'x'.repeat(100); } - // Enqueue enough events to exceed 2MB for (let i = 0; i < 1000; i++) { dispatcher.enqueue({ type: 'track', @@ -382,6 +345,20 @@ describe('PersistentEventQueue', () => { }); }); + describe('deleteSnapshot', () => { + it('delegates to native module', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.deleteSnapshot(); + + expect(mock.deleteSnapshot).toHaveBeenCalled(); + }); + }); + describe('rehydrate + enqueue ordering', () => { it('rehydrated events appear before new events', async () => { const { store } = mockNativeStorage(); @@ -399,7 +376,6 @@ describe('PersistentEventQueue', () => { await pq.rehydrate(); - // Now enqueue new events dispatcher.enqueue({ type: 'track', event: 'new1' } as any); const queue = dispatcher.getQueueRef(); @@ -497,7 +473,6 @@ describe('PersistentEventQueue', () => { const rehydrated = dispatcher.getQueueRef()[0] as any; expect(rehydrated.sentAt).toBeUndefined(); - // Original timestamp preserved expect(rehydrated.timestamp).toBe(oneDayAgo); }); }); @@ -561,33 +536,29 @@ describe('PersistentEventQueue', () => { }); }); - describe('flushEventsToOverflowDisk', () => { - it('writes events to overflow disk store', async () => { + describe('flushEventsToDisk', () => { + it('writes events to disk store', async () => { const { store } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - pq.flushEventsToOverflowDisk([ + await pq.flushEventsToDisk([ { type: 'track', event: 'o1' }, { type: 'track', event: 'o2' }, ]); - // Allow async write to complete - await new Promise((resolve) => setTimeout(resolve, 50)); - - const written = JSON.parse(store.overflow!); + const written = JSON.parse(store.data!); expect(written.version).toBe(1); expect(written.events.length).toBe(2); expect(written.events[0].event).toBe('o1'); }); - it('merges with existing overflow on disk', async () => { + it('merges with existing events on disk', async () => { const { store } = mockNativeStorage(); - // Pre-populate overflow on disk - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [{ type: 'track', event: 'existing1' }], }); @@ -596,11 +567,9 @@ describe('PersistentEventQueue', () => { const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - pq.flushEventsToOverflowDisk([{ type: 'track', event: 'new1' }]); + await pq.flushEventsToDisk([{ type: 'track', event: 'new1' }]); - await new Promise((resolve) => setTimeout(resolve, 50)); - - const written = JSON.parse(store.overflow!); + const written = JSON.parse(store.data!); expect(written.events.length).toBe(2); expect(written.events[0].event).toBe('existing1'); expect(written.events[1].event).toBe('new1'); @@ -609,8 +578,7 @@ describe('PersistentEventQueue', () => { it('enforces disk cap when merging', async () => { const { store } = mockNativeStorage(); - // Pre-populate with 4 events on disk - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: Array.from({ length: 4 }, (_, i) => ({ type: 'track', @@ -621,20 +589,17 @@ describe('PersistentEventQueue', () => { const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher, { - maxOfflineDiskEvents: 5, + maxDiskEvents: 5, }); - pq.flushEventsToOverflowDisk([ + await pq.flushEventsToDisk([ { type: 'track', event: 'new1' }, { type: 'track', event: 'new2' }, ]); - await new Promise((resolve) => setTimeout(resolve, 50)); - - const written = JSON.parse(store.overflow!); + const written = JSON.parse(store.data!); // 4 + 2 = 6, cap = 5, drop 1 oldest expect(written.events.length).toBe(5); - // oldest (old0) should be dropped expect(written.events[0].event).toBe('old1'); }); @@ -645,23 +610,36 @@ describe('PersistentEventQueue', () => { const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - pq.flushEventsToOverflowDisk([]); + await pq.flushEventsToDisk([]); - await new Promise((resolve) => setTimeout(resolve, 50)); + expect(mock.writeSnapshot).not.toHaveBeenCalled(); + }); + + it('serializes concurrent writes (no races)', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); - expect(mock.writeOverflowSnapshot).not.toHaveBeenCalled(); + const p1 = pq.flushEventsToDisk([{ type: 'track', event: 'a' }]); + const p2 = pq.flushEventsToDisk([{ type: 'track', event: 'b' }]); + + await Promise.all([p1, p2]); + + expect(mock.writeSnapshot).toHaveBeenCalledTimes(2); }); }); describe('drainDiskToNetwork', () => { - it('sends overflow events directly to network without entering memory queue', async () => { + it('sends disk events directly to network without entering memory queue', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [ - { type: 'track', event: 'overflow1' }, - { type: 'track', event: 'overflow2' }, + { type: 'track', event: 'disk1' }, + { type: 'track', event: 'disk2' }, ], }); @@ -671,15 +649,14 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // Events sent via sendBatchDirect (fetchWithTimeout) - expect(dispatcher.getQueueRef().length).toBe(0); // memory queue untouched + expect(dispatcher.getQueueRef().length).toBe(0); expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalled(); }); - it('deletes overflow file after drain', async () => { + it('deletes disk file after drain', async () => { const { store, mock } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [{ type: 'track', event: 'e1' }], }); @@ -690,14 +667,14 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); - expect(store.overflow).toBeNull(); + expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(store.data).toBeNull(); }); it('stops on 5xx server error', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: Array.from({ length: 5 }, (_, i) => ({ type: 'track', @@ -717,16 +694,15 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // Events should still be on disk - expect(store.overflow).not.toBeNull(); - const remaining = JSON.parse(store.overflow!); + expect(store.data).not.toBeNull(); + const remaining = JSON.parse(store.data!); expect(remaining.events.length).toBe(5); }); it('stops on 429 rate limit', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [{ type: 'track', event: 'e1' }], }); @@ -742,13 +718,13 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - expect(store.overflow).not.toBeNull(); + expect(store.data).not.toBeNull(); }); it('halves batch size on 413 and retries', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: Array.from({ length: 200 }, (_, i) => ({ type: 'track', @@ -762,7 +738,6 @@ describe('PersistentEventQueue', () => { callCount++; const batchLen = JSON.parse(init.body).batch.length; batchSizes.push(batchLen); - // First call: 413 at batch size 100 if (callCount === 1) { return { ok: false, status: 413 }; } @@ -775,18 +750,14 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // Call 1: 100 (413) — halve to 50 - // Call 2: 50 (success) — batch restores to 100, 150 remaining - // Call 3: 100 (success) — 50 remaining - // Call 4: 50 (success) — done expect(batchSizes).toEqual([100, 50, 100, 50]); - expect(store.overflow).toBeNull(); + expect(store.data).toBeNull(); }); it('drops events on 413 at batchSize=1', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [ { type: 'track', event: 'big_event', messageId: 'msg1' }, @@ -794,7 +765,6 @@ describe('PersistentEventQueue', () => { ], }); - // Always return 413 const dispatcher = createDispatcher({ fetchWithTimeout: jest.fn(async () => ({ ok: false, @@ -806,14 +776,13 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // All events should be dropped (413 at batchSize=1 drops them) - expect(store.overflow).toBeNull(); + expect(store.data).toBeNull(); }); - it('deletes overflow store on fatal config error (401/403/404)', async () => { + it('deletes disk store on fatal config error (401/403/404)', async () => { const { store, mock } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [{ type: 'track', event: 'e1' }], }); @@ -829,14 +798,14 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); - expect(store.overflow).toBeNull(); + expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(store.data).toBeNull(); }); it('drops batch on other 4xx and continues draining', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: Array.from({ length: 150 }, (_, i) => ({ type: 'track', @@ -848,10 +817,8 @@ describe('PersistentEventQueue', () => { const fetchMock = jest.fn(async () => { callCount++; if (callCount === 1) { - // First batch: 400 error — drop it and continue return { ok: false, status: 400 }; } - // Second batch: success return { ok: true, status: 200 }; }); @@ -861,15 +828,14 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // First batch dropped (100 events), second batch sent (50 events) expect(fetchMock).toHaveBeenCalledTimes(2); - expect(store.overflow).toBeNull(); + expect(store.data).toBeNull(); }); it('restores batch size after success following 413', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: Array.from({ length: 300 }, (_, i) => ({ type: 'track', @@ -893,17 +859,15 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // First: 100 (413), then 50 (success, restore), then 100 (success), then 100 (success) - expect(batchSizes[0]).toBe(100); // 413 - expect(batchSizes[1]).toBe(50); // halved - // After success, batch size doubles back toward 100 + expect(batchSizes[0]).toBe(100); + expect(batchSizes[1]).toBe(50); expect(batchSizes[2]).toBe(100); }); it('drains in batches of 100', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: Array.from({ length: 250 }, (_, i) => ({ type: 'track', @@ -924,12 +888,11 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // Should have sent 3 batches: 100 + 100 + 50 expect(fetchCalls).toEqual([100, 100, 50]); - expect(store.overflow).toBeNull(); + expect(store.data).toBeNull(); }); - it('is a no-op when no overflow file exists', async () => { + it('is a no-op when no disk file exists', async () => { mockNativeStorage(); const dispatcher = createDispatcher(); @@ -942,7 +905,7 @@ describe('PersistentEventQueue', () => { }); it('filters expired events during drain', async () => { - const { store: overflowStore } = mockNativeStorage(); + const { store } = mockNativeStorage(); const eightDaysAgo = new Date( Date.now() - 8 * 24 * 60 * 60 * 1000 @@ -951,7 +914,7 @@ describe('PersistentEventQueue', () => { Date.now() - 1 * 24 * 60 * 60 * 1000 ).toISOString(); - overflowStore.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [ { type: 'track', event: 'expired', timestamp: eightDaysAgo }, @@ -979,7 +942,7 @@ describe('PersistentEventQueue', () => { it('stops on network/transport error (null response)', async () => { const { store } = mockNativeStorage(); - store.overflow = JSON.stringify({ + store.data = JSON.stringify({ version: 1, events: [{ type: 'track', event: 'e1' }], }); @@ -994,8 +957,7 @@ describe('PersistentEventQueue', () => { await pq.drainDiskToNetwork(dispatcher); - // Events should still be on disk - expect(store.overflow).not.toBeNull(); + expect(store.data).not.toBeNull(); }); }); @@ -1010,7 +972,6 @@ describe('PersistentEventQueue', () => { } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - // Enqueue events for (let i = 0; i < 5; i++) { dispatcher.enqueue({ type: 'track', @@ -1019,10 +980,8 @@ describe('PersistentEventQueue', () => { } as any); } - // Flush to disk await pq.flushToDisk(); - // Simulate process restart: new dispatcher, reset guard _resetRehydrationGuard(); const dispatcher2 = createDispatcher(); const pq2 = new PersistentEventQueue(dispatcher2); @@ -1051,25 +1010,19 @@ describe('PersistentEventQueue', () => { } as any); } - // Flush network drains events await dispatcher.flush(); - // Queue should be empty after successful network flush expect(dispatcher.getQueueRef().length).toBe(0); - // Add 2 more events dispatcher.enqueue({ type: 'track', event: 'late1' } as any); dispatcher.enqueue({ type: 'track', event: 'late2' } as any); - // Flush to disk await pq.flushToDisk(); - // Simulate restart _resetRehydrationGuard(); const dispatcher2 = createDispatcher(); const pq2 = new PersistentEventQueue(dispatcher2); await pq2.rehydrate(); - // Should only have the 2 late events expect(dispatcher2.getQueueRef().length).toBe(2); expect((dispatcher2.getQueueRef()[0] as any).event).toBe('late1'); }); diff --git a/src/analytics/persistence/__tests__/integration.test.ts b/src/analytics/persistence/__tests__/integration.test.ts index bc3871a..7576ca0 100644 --- a/src/analytics/persistence/__tests__/integration.test.ts +++ b/src/analytics/persistence/__tests__/integration.test.ts @@ -2,10 +2,7 @@ import { NativeModules, AppState } from 'react-native'; import { _resetRehydrationGuard } from '../PersistentEventQueue'; function mockNativeStorage() { - const store: { data: string | null; overflow: string | null } = { - data: null, - overflow: null, - }; + const store: { data: string | null } = { data: null }; NativeModules.MetaRouterQueueStorage = { readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { @@ -14,13 +11,6 @@ function mockNativeStorage() { deleteSnapshot: jest.fn(async () => { store.data = null; }), - readOverflowSnapshot: jest.fn(async () => store.overflow), - writeOverflowSnapshot: jest.fn(async (data: string) => { - store.overflow = data; - }), - deleteOverflowSnapshot: jest.fn(async () => { - store.overflow = null; - }), }; return { store, mock: NativeModules.MetaRouterQueueStorage }; } @@ -113,7 +103,7 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { expect(mock.writeSnapshot).toHaveBeenCalled(); }); - it('skips disk write when network flush succeeds on background', async () => { + it('leaves disk untouched when network flush succeeds on background with empty memory', async () => { const { mock } = mockNativeStorage(); const { @@ -129,12 +119,13 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { client.track('event1', { key: 'value' }); client.track('event2', { key: 'value' }); - // Simulate app going to background — flush succeeds, queue empty + // Simulate app going to background — flush succeeds, queue empty, no disk write await handleAppStateChange!('background'); - // Queue was drained by network flush, so deleteSnapshot is called (empty queue) + // Queue was drained by network flush — post-consolidation, flushToDisk is + // a no-op when memory is empty (doesn't touch disk state). expect(mock.writeSnapshot).not.toHaveBeenCalled(); - expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(mock.deleteSnapshot).not.toHaveBeenCalled(); }); it('deletes snapshot on reset', async () => { @@ -154,7 +145,7 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { expect(mock.deleteSnapshot).toHaveBeenCalled(); }); - it('overflow events written to overflow disk when memory queue overflows', async () => { + it('capacity overflow writes events to disk', async () => { const { mock } = mockNativeStorage(); // Network fails so events stay in queue @@ -188,8 +179,8 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { // Allow async overflow writes to complete await new Promise((resolve) => setTimeout(resolve, 100)); - // Overflow should have been written to disk - expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + // Overflow should have been written to the single disk file + expect(mock.writeSnapshot).toHaveBeenCalled(); }); it('events enqueue successfully while offline (no errors, no HTTP attempts)', async () => { @@ -214,30 +205,12 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { client.track('offline_event_1'); client.track('offline_event_2'); - // Flush should not make any HTTP calls (events go to offline storage) + // Flush should not make any HTTP calls (events go to disk) await client.flush(); expect(global.fetch).not.toHaveBeenCalled(); }); - it('reset deletes both queue and overflow snapshots', async () => { - const { mock } = mockNativeStorage(); - - const { - MetaRouterAnalyticsClient, - } = require('../../MetaRouterAnalyticsClient'); - const client = new MetaRouterAnalyticsClient({ - writeKey: 'test-key', - ingestionHost: 'https://example.com', - }); - await client.init(); - - await client.reset(); - - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(mock.deleteOverflowSnapshot).toHaveBeenCalled(); - }); - - it('offline flush sends events to overflow disk, online drains them', async () => { + it('offline flush sends events to disk, online drains them', async () => { const { mock } = mockNativeStorage(); const { @@ -259,16 +232,16 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { client.track('offline_1'); client.track('offline_2'); - // Flush while offline — events should go to overflow disk + // Flush while offline — events should go to disk await client.flush(); - // Allow async overflow writes to complete + // Allow async disk writes to complete await new Promise((resolve) => setTimeout(resolve, 100)); - expect(mock.writeOverflowSnapshot).toHaveBeenCalled(); + expect(mock.writeSnapshot).toHaveBeenCalled(); expect(global.fetch).not.toHaveBeenCalled(); - // Go online — should drain overflow disk to network + // Go online — should drain disk to network monitor.simulate('connected'); // Allow drain to complete diff --git a/src/analytics/persistence/types.ts b/src/analytics/persistence/types.ts index 31d5ea1..9f398d4 100644 --- a/src/analytics/persistence/types.ts +++ b/src/analytics/persistence/types.ts @@ -21,5 +21,5 @@ export const FLUSH_THRESHOLD_BYTES = 2 * 1024 * 1024; /** Flush-to-disk threshold: event count */ export const FLUSH_THRESHOLD_COUNT = 500; -/** Default max events stored on disk during extended offline periods */ -export const DEFAULT_MAX_OFFLINE_DISK_EVENTS = 10_000; +/** Default max events stored on disk (crash safety + extended offline overflow) */ +export const DEFAULT_MAX_DISK_EVENTS = 10_000; From 23b554e8bb90dbf91ab1f0fc4cc04e61286f9f49 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Thu, 16 Apr 2026 19:47:06 -0600 Subject: [PATCH 04/10] feat: cheap hasDiskData check on boot, drain disk to network directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces rehydrate-into-memory with a cheap file-existence probe so a large on-disk backlog no longer forces an allocation spike on cold start. Memory queue stays empty through init; disk events are drained directly to the network by the dispatcher when online. - Native (iOS/Android): new exists() method returning a boolean without reading file contents - PersistentEventQueue: rehydrate() removed, replaced by checkForPersistedEvents() which sets the hasDiskData flag from the cheap exists probe; hasDiskData is kept in sync on write/delete/drain - MetaRouterAnalyticsClient.init(): calls checkForPersistedEvents instead of rehydrate; triggers drainDiskToNetwork only when online AND hasDiskData is true (no unconditional cold-start flush) - getDebugInfo exposes hasDiskData (boolean) in place of rehydratedEvents (count); we no longer parse the file just to report its size - TTL filter: preserve events with unparseable timestamps (conservative, matches iOS/Android) — previously they were silently dropped --- .../MetaRouterQueueStorageModule.kt | 14 + ios/MetaRouterQueueStorage.m | 13 + src/analytics/MetaRouterAnalyticsClient.ts | 22 +- .../persistence/NativeQueueStorage.ts | 12 + .../persistence/PersistentEventQueue.ts | 136 ++--- .../__tests__/PersistentEventQueue.test.ts | 471 ++++-------------- .../persistence/__tests__/integration.test.ts | 60 ++- 7 files changed, 237 insertions(+), 491 deletions(-) diff --git a/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt b/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt index d92eabd..080cfb9 100644 --- a/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt +++ b/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt @@ -40,6 +40,20 @@ class MetaRouterQueueStorageModule( } } + /** + * Cheap existence check — does not read the file contents. + * Used on boot so a large snapshot doesn't get fully parsed just to + * discover whether there's anything to drain. + */ + @ReactMethod + fun exists(promise: Promise) { + try { + promise.resolve(snapshotFile().exists()) + } catch (e: Exception) { + promise.reject("EXISTS_ERROR", "Failed to check snapshot existence", e) + } + } + @ReactMethod fun readSnapshot(promise: Promise) { try { diff --git a/ios/MetaRouterQueueStorage.m b/ios/MetaRouterQueueStorage.m index f65bf76..4c9e48a 100644 --- a/ios/MetaRouterQueueStorage.m +++ b/ios/MetaRouterQueueStorage.m @@ -48,6 +48,19 @@ - (BOOL)ensureDirectoryExists:(NSError **)error { error:error]; } +/** + * Cheap existence check — does not read the file contents. + * Used on boot so a large snapshot doesn't get fully parsed just to + * discover whether there's anything to drain. + */ +RCT_EXPORT_METHOD(exists:(RCTPromiseResolveBlock)resolve + rejecter:(RCTPromiseRejectBlock)reject) +{ + NSString *path = [self snapshotPath]; + NSFileManager *fm = [NSFileManager defaultManager]; + resolve(@([fm fileExistsAtPath:path])); +} + RCT_EXPORT_METHOD(readSnapshot:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index 7177c75..d85dd15 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -169,7 +169,11 @@ export class MetaRouterAnalyticsClient { try { await this.identityManager.init(); - await this.persistentQueue.rehydrate(); + // Cheap existence check — memory queue starts empty. Any on-disk + // events are drained directly to the network below (if online) via + // drainDiskToNetwork, avoiding a memory spike from rehydrating a + // potentially-large backlog. + await this.persistentQueue.checkForPersistedEvents(); this.startFlushLoop(); this.setupAppStateListener(); @@ -196,7 +200,7 @@ export class MetaRouterAnalyticsClient { this.dispatcher.resetCircuitBreaker(); // (1) Memory queue → network (existing path) void this.flush(); - // (2) Disk overflow → network directly (no rehydration into memory queue) + // (2) Disk → network directly (no load into memory queue) void this.persistentQueue.drainDiskToNetwork(this.dispatcher); } else if (status === 'disconnected') { log('Network connectivity lost — pausing HTTP attempts'); @@ -206,12 +210,12 @@ export class MetaRouterAnalyticsClient { log('MetaRouter SDK initialized'); - // Flush immediately so rehydrated events ship on cold start - // (AppState 'change' won't fire because the app is already active) - void this.flush(); - - // If online at launch, drain any overflow from a previous session - if (this.networkStatus === 'connected') { + // If online at launch with on-disk events, drain them to the network. + // Memory queue starts empty, so there's no cold-start flush to run. + if ( + this.networkStatus === 'connected' && + this.persistentQueue.hasDiskData + ) { void this.persistentQueue.drainDiskToNetwork(this.dispatcher); } } catch (error) { @@ -508,7 +512,7 @@ export class MetaRouterAnalyticsClient { circuitRemainingMs: d.circuitRemainingMs, maxQueueBytes: d.maxQueueBytes, tracingEnabled: this.tracingEnabled, - rehydratedEvents: this.persistentQueue.rehydratedEvents, + hasDiskData: this.persistentQueue.hasDiskData, networkStatus: this.networkStatus, }; } diff --git a/src/analytics/persistence/NativeQueueStorage.ts b/src/analytics/persistence/NativeQueueStorage.ts index f757ba7..01f21d5 100644 --- a/src/analytics/persistence/NativeQueueStorage.ts +++ b/src/analytics/persistence/NativeQueueStorage.ts @@ -13,6 +13,7 @@ import { warn } from '../utils/logger'; * native so the policy stays in one place and is easy to test. */ interface NativeQueueStorageModule { + exists(): Promise; readSnapshot(): Promise; writeSnapshot(data: string): Promise; deleteSnapshot(): Promise; @@ -31,6 +32,17 @@ function getModule(): NativeQueueStorageModule | null { return mod; } +export async function exists(): Promise { + const mod = getModule(); + if (!mod?.exists) return false; + try { + return await mod.exists(); + } catch (err) { + warn('Failed to check queue snapshot existence:', err); + return false; + } +} + export async function readSnapshot(): Promise { const mod = getModule(); if (!mod) return null; diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index 34fdbdc..485256b 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -1,6 +1,7 @@ import type { EventPayload } from '../types'; import type Dispatcher from '../dispatcher'; import { + exists as nativeExists, readSnapshot, writeSnapshot, deleteSnapshot as nativeDeleteSnapshot, @@ -19,13 +20,6 @@ import { } from '../utils/responseCategory'; import { log, warn } from '../utils/logger'; -/** - * Module-level guard: rehydrate at most once per process lifetime. - * Prevents duplicate rehydration if init() is called multiple times - * (e.g. after a reset + reinit in the same session). - */ -let hasRehydrated = false; - /** * Default batch size for draining disk events to network. * May be halved on 413 responses. @@ -39,11 +33,16 @@ const DRAIN_BATCH_SIZE = 100; * and offline overflow. All writes go through {@link flushEventsToDisk} which * reads existing disk contents, merges the new events, enforces the disk cap, * and atomically writes back. + * + * On boot, {@link checkForPersistedEvents} does a cheap existence check and + * sets {@link hasDiskData}. The in-memory queue starts empty; when online, + * the dispatcher drains disk events directly to the network via + * {@link drainDiskToNetwork} instead of rehydrating into memory. */ export class PersistentEventQueue { private readonly dispatcher: Dispatcher; private readonly maxDiskEvents: number; - private _rehydratedEvents: number = 0; + private _hasDiskData: boolean = false; /** Serializes all disk writes (flushEventsToDisk, flushToDisk). */ private diskWriteInFlight: Promise | null = null; @@ -55,73 +54,28 @@ export class PersistentEventQueue { this.maxDiskEvents = opts?.maxDiskEvents ?? DEFAULT_MAX_DISK_EVENTS; } - get rehydratedEvents(): number { - return this._rehydratedEvents; + /** True when a disk snapshot existed at boot or was written since. */ + get hasDiskData(): boolean { + return this._hasDiskData; } /** - * Rehydrate events from disk into the in-memory queue. - * Only runs once per process lifetime. - * - * (NOTE: C3 will replace this with a cheap exists() check + hasDiskData flag - * so disk events drain directly to network instead of being loaded into memory.) + * Boot-time cheap existence check. + * Sets {@link hasDiskData} without reading or parsing the snapshot file, + * so we can decide whether to trigger a drain without paying the cost of + * loading a potentially-large snapshot into memory. */ - async rehydrate(): Promise { - if (hasRehydrated) { - log('Rehydration already completed this process — skipping'); - return; - } - hasRehydrated = true; - + async checkForPersistedEvents(): Promise { try { - const raw = await readSnapshot(); - if (!raw) { - log('No queue snapshot found on disk'); - return; - } - - let snapshot: QueueSnapshot; - try { - snapshot = JSON.parse(raw); - } catch { - warn('Queue snapshot is corrupt JSON — discarding'); - await nativeDeleteSnapshot(); - return; + this._hasDiskData = await nativeExists(); + if (this._hasDiskData) { + log('Persisted events detected on disk — drain pending'); } - - if (snapshot.version !== SNAPSHOT_VERSION) { - warn( - `Queue snapshot version ${snapshot.version} is not supported (expected ${SNAPSHOT_VERSION}) — discarding` - ); - await nativeDeleteSnapshot(); - return; - } - - if (!Array.isArray(snapshot.events) || snapshot.events.length === 0) { - log('Queue snapshot has no events — discarding'); - await nativeDeleteSnapshot(); - return; - } - - const fresh = filterExpired(snapshot.events); - const expired = snapshot.events.length - fresh.length; - if (expired > 0) { - warn(`Dropped ${expired} expired events (older than 7 days)`); - } - - if (fresh.length === 0) { - log('All snapshot events expired — discarding'); - await nativeDeleteSnapshot(); - return; - } - - this.dispatcher.enqueueFront(fresh); - this._rehydratedEvents = fresh.length; - log(`Rehydrated ${fresh.length} events from disk`); - - await nativeDeleteSnapshot(); + return this._hasDiskData; } catch (err) { - warn('Failed to rehydrate queue from disk:', err); + warn('Failed to check for persisted events:', err); + this._hasDiskData = false; + return false; } } @@ -132,10 +86,7 @@ export class PersistentEventQueue { */ async flushToDisk(): Promise { const queue = this.dispatcher.getQueueRef(); - if (queue.length === 0) { - // Nothing in memory to persist. Leave any existing disk state alone. - return; - } + if (queue.length === 0) return; const events = this.dispatcher.drainQueue(); return this.flushEventsToDisk(events); } @@ -148,7 +99,6 @@ export class PersistentEventQueue { async flushEventsToDisk(events: EventPayload[]): Promise { if (events.length === 0) return; - // Chain onto any in-flight write so writes are ordered. const prev = this.diskWriteInFlight; const next = (async () => { if (prev) await prev; @@ -165,7 +115,6 @@ export class PersistentEventQueue { const existing = await this._readExistingEvents(); let combined = existing.concat(events); - // Enforce disk cap by dropping oldest if (combined.length > this.maxDiskEvents) { const dropCount = combined.length - this.maxDiskEvents; combined = combined.slice(dropCount); @@ -174,6 +123,7 @@ export class PersistentEventQueue { if (combined.length === 0) { await nativeDeleteSnapshot(); + this._hasDiskData = false; return; } @@ -182,6 +132,7 @@ export class PersistentEventQueue { events: combined, }; await writeSnapshot(JSON.stringify(snapshot)); + this._hasDiskData = true; log( `Memory queue flushed to disk: ${events.length} events, ${combined.length} total on disk` ); @@ -201,12 +152,12 @@ export class PersistentEventQueue { } /** - * Drain disk events directly to network in batches. - * Does NOT load events into the memory queue. - * Called on offline→online transition and after successful online flushes. + * Drain disk events directly to network in batches. Does NOT load events + * into the memory queue. Called on offline→online transition and after + * successful online flushes. * * Response handling: - * - 200-299: advances and deletes sent events; restores batch size after 413 + * - 2xx: advances and deletes sent events; restores batch size after 413 * - 413: halves batch size, retries. Drops at batchSize=1 * - 5xx/408/429: stops, writes remainder back, retries on next online transition * - 401/403/404: fatal, deletes disk store @@ -227,7 +178,10 @@ export class PersistentEventQueue { try { while (true) { const raw = await readSnapshot(); - if (!raw) return; + if (!raw) { + this._hasDiskData = false; + return; + } let snapshot: QueueSnapshot; try { @@ -235,11 +189,13 @@ export class PersistentEventQueue { } catch { warn('Queue snapshot is corrupt during drain — deleting'); await nativeDeleteSnapshot(); + this._hasDiskData = false; return; } if (!Array.isArray(snapshot.events) || snapshot.events.length === 0) { await nativeDeleteSnapshot(); + this._hasDiskData = false; return; } @@ -254,17 +210,16 @@ export class PersistentEventQueue { if (events.length === 0) { log('All disk events expired — deleting'); await nativeDeleteSnapshot(); + this._hasDiskData = false; return; } - // Take a batch from the front (oldest first) const n = Math.min(batchSize, events.length); const batch = events.slice(0, n); const remaining = events.slice(n); const response = await dispatcher.sendBatchDirect(batch); - // Network/transport error — stop, retry on next online transition if (!response) { log( `Disk drain paused — network error, ${events.length} event(s) remain on disk` @@ -287,6 +242,7 @@ export class PersistentEventQueue { if (remaining.length === 0) { await nativeDeleteSnapshot(); + this._hasDiskData = false; log('Disk store drain complete'); } else { await writeSnapshot( @@ -311,7 +267,6 @@ export class PersistentEventQueue { JSON.stringify({ version: SNAPSHOT_VERSION, events }) ); } - // Continue loop with smaller batch } else { const ids = (batch as any[]) .map((e) => (e as any).messageId) @@ -321,6 +276,7 @@ export class PersistentEventQueue { ); if (remaining.length === 0) { await nativeDeleteSnapshot(); + this._hasDiskData = false; } else { await writeSnapshot( JSON.stringify({ @@ -351,6 +307,7 @@ export class PersistentEventQueue { `Disk drain: fatal config error ${response.statusCode} — deleting disk store` ); await nativeDeleteSnapshot(); + this._hasDiskData = false; return; } @@ -360,6 +317,7 @@ export class PersistentEventQueue { ); if (remaining.length === 0) { await nativeDeleteSnapshot(); + this._hasDiskData = false; } else { await writeSnapshot( JSON.stringify({ @@ -382,6 +340,7 @@ export class PersistentEventQueue { */ async deleteSnapshot(): Promise { await nativeDeleteSnapshot(); + this._hasDiskData = false; } private async _readExistingEvents(): Promise { @@ -409,14 +368,9 @@ function filterExpired(events: EventPayload[]): EventPayload[] { const ts = (e as any).timestamp; if (!ts) return true; const eventTime = new Date(ts).getTime(); - return !isNaN(eventTime) && eventTime > cutoff; + // Unparseable timestamp — keep it. Conservative: better to send than to + // silently drop an event whose age we can't verify. + if (isNaN(eventTime)) return true; + return eventTime > cutoff; }); } - -/** - * Reset the rehydration guard. Exposed ONLY for testing. - * @internal - */ -export function _resetRehydrationGuard(): void { - hasRehydrated = false; -} diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 369c686..19815dc 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -5,6 +5,7 @@ import CircuitBreaker from '../../utils/circuitBreaker'; function mockNativeStorage() { const store: { data: string | null } = { data: null }; NativeModules.MetaRouterQueueStorage = { + exists: jest.fn(async () => store.data !== null), readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { store.data = data; @@ -48,195 +49,73 @@ function createDispatcher(overrides?: Partial) { describe('PersistentEventQueue', () => { beforeEach(() => { jest.clearAllMocks(); - const { _resetRehydrationGuard } = require('../PersistentEventQueue'); - _resetRehydrationGuard(); }); afterEach(() => { NativeModules.MetaRouterQueueStorage = undefined as any; }); - describe('rehydrate', () => { - it('loads events from disk on first call and prepends to queue', async () => { + describe('checkForPersistedEvents', () => { + it('sets hasDiskData=true when the snapshot file exists', async () => { const { store } = mockNativeStorage(); - const events = [ - { type: 'track', event: 'disk1', messageId: '1' }, - { type: 'track', event: 'disk2', messageId: '2' }, - ]; - store.data = JSON.stringify({ version: 1, events }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(2); - expect((dispatcher.getQueueRef()[0] as any).event).toBe('disk1'); - }); - - it('does not rehydrate a second time', async () => { - const { store, mock } = mockNativeStorage(); store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'disk1' }], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - await pq.rehydrate(); - - expect(mock.readSnapshot).toHaveBeenCalledTimes(1); - expect(dispatcher.getQueueRef().length).toBe(1); - }); - - it('flushes entire queue to disk when rehydrated events exceed memory byte cap', async () => { - const { store } = mockNativeStorage(); - // Use uniform-size events so byte cap maps to exact count - const events = Array.from({ length: 2500 }, (_, i) => ({ - type: 'track', - event: 'evt', - properties: { id: String(i).padStart(5, '0') }, - })); - const eventSize = JSON.stringify(events[0]).length; - store.data = JSON.stringify({ version: 1, events }); - - const overflowEvents: any[] = []; - const dispatcher = createDispatcher({ - maxQueueBytes: eventSize * 2000, - onCapacityOverflow: (evts: any[]) => { - overflowEvents.push(...evts); - }, - }); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(0); - expect(overflowEvents.length).toBe(2500); - }); - - it('skips rehydration if snapshot has unknown version', async () => { - const { store } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 99, - events: [{ type: 'track', event: 'x' }], + events: [{ type: 'track', event: 'e1' }], }); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - + const result = await pq.checkForPersistedEvents(); + expect(result).toBe(true); + expect(pq.hasDiskData).toBe(true); + // Memory queue stays empty — no load on boot. expect(dispatcher.getQueueRef().length).toBe(0); }); - it('handles corrupt JSON gracefully', async () => { - const { store } = mockNativeStorage(); - store.data = '{not valid json'; - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(0); - }); - - it('handles null snapshot (no file on disk)', async () => { + it('sets hasDiskData=false when there is no snapshot', async () => { mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(0); + const result = await pq.checkForPersistedEvents(); + expect(result).toBe(false); + expect(pq.hasDiskData).toBe(false); }); - it('deletes snapshot after successful rehydration', async () => { + it('does not read the snapshot file (cheap exists-only check)', async () => { const { store, mock } = mockNativeStorage(); store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'x' }], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - expect(mock.deleteSnapshot).toHaveBeenCalled(); - }); - - it('deletes snapshot when version is unknown', async () => { - const { store, mock } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 99, - events: [{ type: 'track', event: 'x' }], + events: [{ type: 'track', event: 'e1' }], }); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.checkForPersistedEvents(); - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(dispatcher.getQueueRef().length).toBe(0); + expect(mock.exists).toHaveBeenCalled(); + expect(mock.readSnapshot).not.toHaveBeenCalled(); }); - it('deletes snapshot when JSON is corrupt', async () => { - const { store, mock } = mockNativeStorage(); - store.data = '{not valid json'; + it('returns false if the exists check throws', async () => { + mockNativeStorage(); + ( + NativeModules.MetaRouterQueueStorage as any + ).exists.mockRejectedValueOnce(new Error('boom')); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(dispatcher.getQueueRef().length).toBe(0); - }); - }); - - describe('resetRehydrationGuard', () => { - it('allows rehydration again after reset', async () => { - const { store, mock } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 1, - events: [{ type: 'track', event: 'e1' }], - }); - - const dispatcher = createDispatcher(); - const { - PersistentEventQueue, - _resetRehydrationGuard, - } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - expect(dispatcher.getQueueRef().length).toBe(1); - - _resetRehydrationGuard(); - store.data = JSON.stringify({ - version: 1, - events: [{ type: 'track', event: 'e2' }], - }); - - const pq2 = new PersistentEventQueue(dispatcher); - await pq2.rehydrate(); - - expect(mock.readSnapshot).toHaveBeenCalledTimes(2); + const result = await pq.checkForPersistedEvents(); + expect(result).toBe(false); + expect(pq.hasDiskData).toBe(false); }); }); @@ -266,8 +145,8 @@ describe('PersistentEventQueue', () => { expect(written.version).toBe(1); expect(written.events.length).toBe(2); expect(written.events[0].event).toBe('mem1'); - // Memory queue drained expect(dispatcher.getQueueRef().length).toBe(0); + expect(pq.hasDiskData).toBe(true); }); it('is a no-op when memory queue is empty', async () => { @@ -346,193 +225,22 @@ describe('PersistentEventQueue', () => { }); describe('deleteSnapshot', () => { - it('delegates to native module', async () => { + it('delegates to native module and clears hasDiskData', async () => { const { mock } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.deleteSnapshot(); - - expect(mock.deleteSnapshot).toHaveBeenCalled(); - }); - }); - - describe('rehydrate + enqueue ordering', () => { - it('rehydrated events appear before new events', async () => { - const { store } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 1, - events: [ - { type: 'track', event: 'old1', messageId: '1' }, - { type: 'track', event: 'old2', messageId: '2' }, - ], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - dispatcher.enqueue({ type: 'track', event: 'new1' } as any); - - const queue = dispatcher.getQueueRef(); - expect((queue[0] as any).event).toBe('old1'); - expect((queue[1] as any).event).toBe('old2'); - expect((queue[2] as any).event).toBe('new1'); - }); - }); - - describe('TTL expiry', () => { - it('drops events older than 7 days on rehydrate', async () => { - const { store } = mockNativeStorage(); - const now = Date.now(); - const eightDaysAgo = new Date( - now - 8 * 24 * 60 * 60 * 1000 - ).toISOString(); - const oneDayAgo = new Date(now - 1 * 24 * 60 * 60 * 1000).toISOString(); - - store.data = JSON.stringify({ - version: 1, - events: [ - { type: 'track', event: 'expired', timestamp: eightDaysAgo }, - { type: 'track', event: 'fresh', timestamp: oneDayAgo }, - ], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(1); - expect((dispatcher.getQueueRef()[0] as any).event).toBe('fresh'); - }); - - it('keeps events without a timestamp field', async () => { - const { store } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 1, - events: [{ type: 'track', event: 'no_ts' }], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(1); - }); - - it('discards snapshot and deletes from disk when all events are expired', async () => { - const { store, mock } = mockNativeStorage(); - const eightDaysAgo = new Date( - Date.now() - 8 * 24 * 60 * 60 * 1000 - ).toISOString(); - - store.data = JSON.stringify({ - version: 1, - events: [ - { type: 'track', event: 'old1', timestamp: eightDaysAgo }, - { type: 'track', event: 'old2', timestamp: eightDaysAgo }, - ], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); + // Force hasDiskData=true by flushing something first + dispatcher.enqueue({ type: 'track', event: 'e1' } as any); + await pq.flushToDisk(); + expect(pq.hasDiskData).toBe(true); - await pq.rehydrate(); + await pq.deleteSnapshot(); - expect(dispatcher.getQueueRef().length).toBe(0); expect(mock.deleteSnapshot).toHaveBeenCalled(); - }); - }); - - describe('sentAt on rehydrated events', () => { - it('does not set sentAt at rehydration time (drainBatch stamps it at send time)', async () => { - const { store } = mockNativeStorage(); - const oneDayAgo = new Date( - Date.now() - 24 * 60 * 60 * 1000 - ).toISOString(); - - store.data = JSON.stringify({ - version: 1, - events: [{ type: 'track', event: 'e1', timestamp: oneDayAgo }], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - - const rehydrated = dispatcher.getQueueRef()[0] as any; - expect(rehydrated.sentAt).toBeUndefined(); - expect(rehydrated.timestamp).toBe(oneDayAgo); - }); - }); - - describe('rehydratedEvents count', () => { - it('tracks the number of rehydrated events', async () => { - const { store } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 1, - events: [ - { type: 'track', event: 'e1' }, - { type: 'track', event: 'e2' }, - { type: 'track', event: 'e3' }, - ], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - expect(pq.rehydratedEvents).toBe(0); - await pq.rehydrate(); - expect(pq.rehydratedEvents).toBe(3); - }); - - it('excludes expired events from count', async () => { - const { store } = mockNativeStorage(); - const eightDaysAgo = new Date( - Date.now() - 8 * 24 * 60 * 60 * 1000 - ).toISOString(); - const oneDayAgo = new Date( - Date.now() - 24 * 60 * 60 * 1000 - ).toISOString(); - - store.data = JSON.stringify({ - version: 1, - events: [ - { type: 'track', event: 'expired', timestamp: eightDaysAgo }, - { type: 'track', event: 'fresh1', timestamp: oneDayAgo }, - { type: 'track', event: 'fresh2', timestamp: oneDayAgo }, - ], - }); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - expect(pq.rehydratedEvents).toBe(2); - }); - - it('is 0 when no snapshot exists', async () => { - mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { PersistentEventQueue } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); - - await pq.rehydrate(); - expect(pq.rehydratedEvents).toBe(0); + expect(pq.hasDiskData).toBe(false); }); }); @@ -598,7 +306,6 @@ describe('PersistentEventQueue', () => { ]); const written = JSON.parse(store.data!); - // 4 + 2 = 6, cap = 5, drop 1 oldest expect(written.events.length).toBe(5); expect(written.events[0].event).toBe('old1'); }); @@ -653,7 +360,7 @@ describe('PersistentEventQueue', () => { expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalled(); }); - it('deletes disk file after drain', async () => { + it('deletes disk file after drain and clears hasDiskData', async () => { const { store, mock } = mockNativeStorage(); store.data = JSON.stringify({ @@ -665,10 +372,14 @@ describe('PersistentEventQueue', () => { const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); + await pq.checkForPersistedEvents(); + expect(pq.hasDiskData).toBe(true); + await pq.drainDiskToNetwork(dispatcher); expect(mock.deleteSnapshot).toHaveBeenCalled(); expect(store.data).toBeNull(); + expect(pq.hasDiskData).toBe(false); }); it('stops on 5xx server error', async () => { @@ -939,6 +650,33 @@ describe('PersistentEventQueue', () => { expect(sentBatch[0].event).toBe('fresh'); }); + it('keeps events with unparseable timestamps during drain', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [ + { type: 'track', event: 'no_ts' }, + { type: 'track', event: 'bad_ts', timestamp: 'not-a-date' }, + ], + }); + + let sentBatch: any[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + sentBatch = JSON.parse(init.body).batch; + return { ok: true, status: 200 } as any; + }), + }); + + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(sentBatch.length).toBe(2); + }); + it('stops on network/transport error (null response)', async () => { const { store } = mockNativeStorage(); @@ -959,72 +697,69 @@ describe('PersistentEventQueue', () => { expect(store.data).not.toBeNull(); }); - }); - describe('full round-trip', () => { - it('enqueue → flushToDisk → rehydrate preserves events', async () => { - mockNativeStorage(); - - const dispatcher = createDispatcher(); - const { - PersistentEventQueue, - _resetRehydrationGuard, - } = require('../PersistentEventQueue'); - const pq = new PersistentEventQueue(dispatcher); + it('concurrent drains coalesce into one', async () => { + const { store } = mockNativeStorage(); - for (let i = 0; i < 5; i++) { - dispatcher.enqueue({ + store.data = JSON.stringify({ + version: 1, + events: Array.from({ length: 3 }, (_, i) => ({ type: 'track', event: `e${i}`, - properties: { i }, - } as any); - } - - await pq.flushToDisk(); - - _resetRehydrationGuard(); - const dispatcher2 = createDispatcher(); - const pq2 = new PersistentEventQueue(dispatcher2); + })), + }); - await pq2.rehydrate(); + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); - expect(dispatcher2.getQueueRef().length).toBe(5); - expect((dispatcher2.getQueueRef()[0] as any).event).toBe('e0'); - expect((dispatcher2.getQueueRef()[4] as any).event).toBe('e4'); + const [a, b] = await Promise.all([ + pq.drainDiskToNetwork(dispatcher), + pq.drainDiskToNetwork(dispatcher), + ]); + expect(a).toBe(b); + expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalledTimes( + 1 + ); }); + }); - it('partial drain then flush only persists remaining events', async () => { + describe('full round-trip (persist → drain)', () => { + it('flushToDisk persists events, drainDiskToNetwork sends them', async () => { mockNativeStorage(); const dispatcher = createDispatcher(); - const { - PersistentEventQueue, - _resetRehydrationGuard, - } = require('../PersistentEventQueue'); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); for (let i = 0; i < 5; i++) { dispatcher.enqueue({ type: 'track', event: `e${i}`, + properties: { i }, } as any); } - await dispatcher.flush(); - expect(dispatcher.getQueueRef().length).toBe(0); - - dispatcher.enqueue({ type: 'track', event: 'late1' } as any); - dispatcher.enqueue({ type: 'track', event: 'late2' } as any); - await pq.flushToDisk(); + expect(pq.hasDiskData).toBe(true); - _resetRehydrationGuard(); const dispatcher2 = createDispatcher(); const pq2 = new PersistentEventQueue(dispatcher2); - await pq2.rehydrate(); - expect(dispatcher2.getQueueRef().length).toBe(2); - expect((dispatcher2.getQueueRef()[0] as any).event).toBe('late1'); + await pq2.checkForPersistedEvents(); + expect(pq2.hasDiskData).toBe(true); + + await pq2.drainDiskToNetwork(dispatcher2); + + // All 5 events delivered via a single batch + expect((dispatcher2 as any).opts.fetchWithTimeout).toHaveBeenCalledTimes( + 1 + ); + const body = JSON.parse( + (dispatcher2 as any).opts.fetchWithTimeout.mock.calls[0][1].body + ); + expect(body.batch.length).toBe(5); + expect(pq2.hasDiskData).toBe(false); }); }); }); diff --git a/src/analytics/persistence/__tests__/integration.test.ts b/src/analytics/persistence/__tests__/integration.test.ts index 7576ca0..6da7c4b 100644 --- a/src/analytics/persistence/__tests__/integration.test.ts +++ b/src/analytics/persistence/__tests__/integration.test.ts @@ -1,9 +1,9 @@ import { NativeModules, AppState } from 'react-native'; -import { _resetRehydrationGuard } from '../PersistentEventQueue'; function mockNativeStorage() { const store: { data: string | null } = { data: null }; NativeModules.MetaRouterQueueStorage = { + exists: jest.fn(async () => store.data !== null), readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { store.data = data; @@ -20,7 +20,6 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { beforeEach(() => { jest.clearAllMocks(); - _resetRehydrationGuard(); handleAppStateChange = null; // Mock fetch so flush() doesn't hang on a real network call @@ -41,7 +40,7 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { (global.fetch as jest.Mock).mockRestore?.(); }); - it('rehydrates events from disk during init', async () => { + it('drains on-disk events directly to network during init (no memory rehydrate)', async () => { const { store } = mockNativeStorage(); store.data = JSON.stringify({ version: 1, @@ -66,7 +65,10 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { }); await client.init(); - // Rehydrated events should have been flushed immediately on init + // Allow async drain to run + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Disk events should have been sent via drainDiskToNetwork → fetch expect(global.fetch).toHaveBeenCalled(); const body = JSON.parse((global.fetch as jest.Mock).mock.calls[0][1].body); expect(body.batch).toEqual( @@ -76,10 +78,40 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { ); }); + it('does not load disk events into the memory queue on init (cheap exists-only check)', async () => { + const { store, mock } = mockNativeStorage(); + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'persisted' }], + }); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }, + { networkMonitor: monitor } + ); + await client.init(); + + // Offline, so no drain runs. readSnapshot should not have fired. + expect(mock.exists).toHaveBeenCalled(); + expect(mock.readSnapshot).not.toHaveBeenCalled(); + // Memory queue remains empty — disk events will drain when online. + const debug = await client.getDebugInfo(); + expect(debug.queueLength).toBe(0); + expect(debug.hasDiskData).toBe(true); + }); + it('flushes to disk when app goes to background and network fails', async () => { const { mock } = mockNativeStorage(); - // Simulate network failure so events remain in queue for persistence (global.fetch as jest.Mock).mockImplementation(() => Promise.reject(new Error('Network unavailable')) ); @@ -93,11 +125,9 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { }); await client.init(); - // Track some events client.track('event1', { key: 'value' }); client.track('event2', { key: 'value' }); - // Simulate app going to background — flush fails, events persisted to disk await handleAppStateChange!('background'); expect(mock.writeSnapshot).toHaveBeenCalled(); @@ -115,15 +145,11 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { }); await client.init(); - // Track some events client.track('event1', { key: 'value' }); client.track('event2', { key: 'value' }); - // Simulate app going to background — flush succeeds, queue empty, no disk write await handleAppStateChange!('background'); - // Queue was drained by network flush — post-consolidation, flushToDisk is - // a no-op when memory is empty (doesn't touch disk state). expect(mock.writeSnapshot).not.toHaveBeenCalled(); expect(mock.deleteSnapshot).not.toHaveBeenCalled(); }); @@ -148,7 +174,6 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { it('capacity overflow writes events to disk', async () => { const { mock } = mockNativeStorage(); - // Network fails so events stay in queue (global.fetch as jest.Mock).mockImplementation(() => Promise.reject(new Error('Network unavailable')) ); @@ -163,7 +188,6 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { { writeKey: 'test-key', ingestionHost: 'https://example.com', - // Small queue so we can trigger overflow easily maxQueueBytes: 500, maxOfflineDiskEvents: 100, }, @@ -171,15 +195,12 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { ); await client.init(); - // Track many events to overflow memory queue for (let i = 0; i < 50; i++) { client.track(`event_${i}`, { idx: i }); } - // Allow async overflow writes to complete await new Promise((resolve) => setTimeout(resolve, 100)); - // Overflow should have been written to the single disk file expect(mock.writeSnapshot).toHaveBeenCalled(); }); @@ -201,11 +222,9 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { ); await client.init(); - // Track events while offline client.track('offline_event_1'); client.track('offline_event_2'); - // Flush should not make any HTTP calls (events go to disk) await client.flush(); expect(global.fetch).not.toHaveBeenCalled(); }); @@ -228,23 +247,18 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { ); await client.init(); - // Track events while offline client.track('offline_1'); client.track('offline_2'); - // Flush while offline — events should go to disk await client.flush(); - // Allow async disk writes to complete await new Promise((resolve) => setTimeout(resolve, 100)); expect(mock.writeSnapshot).toHaveBeenCalled(); expect(global.fetch).not.toHaveBeenCalled(); - // Go online — should drain disk to network monitor.simulate('connected'); - // Allow drain to complete await new Promise((resolve) => setTimeout(resolve, 100)); expect(global.fetch).toHaveBeenCalled(); From 18c781d7ba09ff6cd26a0c9055e130ac4963a10b Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Thu, 16 Apr 2026 19:52:24 -0600 Subject: [PATCH 05/10] feat: add debounced network monitor with asymmetric 2s online window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps the raw NetworkMonitor so consumers see: - Offline transitions immediately (pause HTTP ASAP — no wasted retries on a down network) - Online transitions only after 2 seconds of stable connectivity (absorb WiFi/cellular flaps that would otherwise trigger flush attempts that immediately fail) A pending online debounce is cancelled if offline arrives during the wait, and the debounce window resets on repeated raw online signals. MetaRouterAnalyticsClient.init() now wraps the default NetworkMonitor in DebouncedNetworkMonitor. An injected monitor (tests, custom reachability providers) is used as-is, so callers control their own debounce. --- src/analytics/MetaRouterAnalyticsClient.ts | 8 +- .../utils/debouncedNetworkMonitor.test.ts | 139 ++++++++++++++++++ .../utils/debouncedNetworkMonitor.ts | 76 ++++++++++ 3 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 src/analytics/utils/debouncedNetworkMonitor.test.ts create mode 100644 src/analytics/utils/debouncedNetworkMonitor.ts diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index d85dd15..03e6fc9 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -18,6 +18,7 @@ import { type NetworkReachability, type NetworkStatus, } from './utils/networkMonitor'; +import { DebouncedNetworkMonitor } from './utils/debouncedNetworkMonitor'; /** * Analytics client for MetaRouter. @@ -84,7 +85,12 @@ export class MetaRouterAnalyticsClient { setDebugLogging(options.debug ?? false); this.identityManager = new IdentityManager(); - this.networkMonitor = deps?.networkMonitor ?? new NetworkMonitor(); + // Default: wrap the raw native monitor with the asymmetric debounce + // (immediate offline, 2s stable-online). If a caller injects their own + // monitor (e.g. in tests), use it as-is so they control the debounce + // behavior explicitly. + this.networkMonitor = + deps?.networkMonitor ?? new DebouncedNetworkMonitor(new NetworkMonitor()); this.maxQueueBytes = options.maxQueueBytes ?? this.maxQueueBytes; this.dispatcher = new Dispatcher({ maxQueueBytes: this.maxQueueBytes, diff --git a/src/analytics/utils/debouncedNetworkMonitor.test.ts b/src/analytics/utils/debouncedNetworkMonitor.test.ts new file mode 100644 index 0000000..2c5b3fe --- /dev/null +++ b/src/analytics/utils/debouncedNetworkMonitor.test.ts @@ -0,0 +1,139 @@ +import { DebouncedNetworkMonitor } from './debouncedNetworkMonitor'; +import { StubNetworkMonitor } from './networkMonitor'; + +describe('DebouncedNetworkMonitor', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('inherits initial status from the inner monitor', () => { + const inner = new StubNetworkMonitor('connected'); + const debounced = new DebouncedNetworkMonitor(inner); + expect(debounced.currentStatus).toBe('connected'); + + const inner2 = new StubNetworkMonitor('disconnected'); + const debounced2 = new DebouncedNetworkMonitor(inner2); + expect(debounced2.currentStatus).toBe('disconnected'); + }); + + it('fires offline transitions immediately', () => { + const inner = new StubNetworkMonitor('connected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('disconnected'); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith('disconnected'); + expect(debounced.currentStatus).toBe('disconnected'); + }); + + it('debounces online transitions by 2s of stable connectivity', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + + // Not yet fired — still within debounce window. + expect(handler).not.toHaveBeenCalled(); + expect(debounced.currentStatus).toBe('disconnected'); + + jest.advanceTimersByTime(1999); + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(1); + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith('connected'); + expect(debounced.currentStatus).toBe('connected'); + }); + + it('cancels pending online fire if offline arrives during the debounce window', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + jest.advanceTimersByTime(1500); + inner.simulate('disconnected'); + + // No online transition ever fired; no offline transition either + // (we were already at 'disconnected' from the debounced perspective). + jest.advanceTimersByTime(5000); + expect(handler).not.toHaveBeenCalled(); + expect(debounced.currentStatus).toBe('disconnected'); + }); + + it('resets the debounce timer on repeated online signals', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + jest.advanceTimersByTime(1000); + // Another online signal before timer fires — should restart the window. + (inner as any)._currentStatus = 'disconnected'; + inner.simulate('connected'); + jest.advanceTimersByTime(1999); + expect(handler).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1); + expect(handler).toHaveBeenCalledTimes(1); + }); + + it('fires online after offline when debounce elapses', () => { + const inner = new StubNetworkMonitor('connected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('disconnected'); + expect(handler).toHaveBeenNthCalledWith(1, 'disconnected'); + + inner.simulate('connected'); + jest.advanceTimersByTime(2000); + expect(handler).toHaveBeenNthCalledWith(2, 'connected'); + }); + + it('stop() cancels pending online timer and unsubscribes from inner', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + debounced.stop(); + + jest.advanceTimersByTime(5000); + expect(handler).not.toHaveBeenCalled(); + + // Further simulations on inner no longer propagate + inner.simulate('disconnected'); + inner.simulate('connected'); + jest.advanceTimersByTime(5000); + expect(handler).not.toHaveBeenCalled(); + }); + + it('does not fire when inner flips offline before the fire callback runs', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + // Flip inner offline silently (simulate a race: timer is scheduled but + // inner.currentStatus changes before the callback runs). + (inner as any)._currentStatus = 'disconnected'; + + jest.advanceTimersByTime(2000); + expect(handler).not.toHaveBeenCalled(); + expect(debounced.currentStatus).toBe('disconnected'); + }); +}); diff --git a/src/analytics/utils/debouncedNetworkMonitor.ts b/src/analytics/utils/debouncedNetworkMonitor.ts new file mode 100644 index 0000000..5926658 --- /dev/null +++ b/src/analytics/utils/debouncedNetworkMonitor.ts @@ -0,0 +1,76 @@ +import type { NetworkReachability, NetworkStatus } from './networkMonitor'; + +/** + * Wraps a {@link NetworkReachability} with asymmetric debouncing: + * - **offline** transitions fire immediately (pause HTTP ASAP — no point + * burning retries on a down network). + * - **online** transitions wait for {@link ONLINE_DEBOUNCE_MS} of stable + * connectivity before firing, so flaky WiFi/cellular handoffs don't + * trigger spurious flush attempts that immediately fail. + * + * A pending online debounce is cancelled if offline arrives during the wait. + */ +export class DebouncedNetworkMonitor implements NetworkReachability { + static readonly ONLINE_DEBOUNCE_MS = 2000; + + private readonly inner: NetworkReachability; + private handler: ((status: NetworkStatus) => void) | null = null; + private unsubscribeInner: (() => void) | null = null; + private onlineTimer: ReturnType | null = null; + private _currentStatus: NetworkStatus; + + constructor(inner: NetworkReachability) { + this.inner = inner; + this._currentStatus = inner.currentStatus; + + this.unsubscribeInner = inner.onStatusChange((rawStatus) => { + if (rawStatus === 'disconnected') { + this.clearOnlineTimer(); + if (this._currentStatus !== 'disconnected') { + this._currentStatus = 'disconnected'; + this.handler?.('disconnected'); + } + return; + } + + // rawStatus === 'connected': start/reset the debounce window + this.clearOnlineTimer(); + if (this._currentStatus === 'connected') return; + + this.onlineTimer = setTimeout(() => { + this.onlineTimer = null; + // Guard against racing with a late offline transition. + if (this.inner.currentStatus !== 'connected') return; + if (this._currentStatus !== 'connected') { + this._currentStatus = 'connected'; + this.handler?.('connected'); + } + }, DebouncedNetworkMonitor.ONLINE_DEBOUNCE_MS); + }); + } + + get currentStatus(): NetworkStatus { + return this._currentStatus; + } + + onStatusChange(handler: (status: NetworkStatus) => void): () => void { + this.handler = handler; + return () => { + this.handler = null; + }; + } + + stop(): void { + this.clearOnlineTimer(); + this.unsubscribeInner?.(); + this.unsubscribeInner = null; + this.handler = null; + } + + private clearOnlineTimer(): void { + if (this.onlineTimer) { + clearTimeout(this.onlineTimer); + this.onlineTimer = null; + } + } +} From 38e93c71e9287aec4df704742d3fd4bd92dade02 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Thu, 16 Apr 2026 20:08:22 -0600 Subject: [PATCH 06/10] feat: public maxDiskEvents + maxQueueEvents api with validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns the RN public surface with iOS/Android: InitOptions now exposes `maxQueueEvents` (count cap, default 2000) and `maxDiskEvents` (default 10000). `maxQueueBytes` moves to an internal 5MB constant — byte caps stay internal on all platforms until a customer needs to tune them. - InitOptions: add maxQueueEvents, add maxDiskEvents, remove maxQueueBytes, remove maxOfflineDiskEvents - Constructor: throws on negative maxDiskEvents (use 0 to disable); silently clamps maxQueueEvents to >= 1; warns when a non-zero maxDiskEvents is smaller than maxQueueEvents (memory could exceed what disk can preserve on background flush) - Dispatcher: enforces BOTH count AND byte caps on enqueue and enqueueFront. New isPersistenceEnabled callback; when it returns false (maxDiskEvents === 0), capacity overflow becomes a ring buffer (drop oldest one-by-one) instead of splicing the whole queue to a disk handler that would silently drop - Dispatcher.flush() offline branch skips the splice-to-disk callback when persistence is disabled — events stay in memory for the next foreground retry - PersistentEventQueue.flushToDisk / flushEventsToDisk are no-ops when maxDiskEvents === 0 (documented kill-loss tradeoff) - getDebugInfo: exposes maxQueueEvents and maxDiskEvents; drops maxQueueBytes from the returned shape --- .../MetaRouterAnalyticsClient.test.ts | 144 +++++++++++++++++- src/analytics/MetaRouterAnalyticsClient.ts | 44 +++++- src/analytics/dispatcher.test.ts | 2 + src/analytics/dispatcher.ts | 99 ++++++++---- src/analytics/init.test.ts | 20 +-- .../persistence/PersistentEventQueue.ts | 10 ++ .../__tests__/PersistentEventQueue.test.ts | 2 + .../persistence/__tests__/integration.test.ts | 4 +- src/analytics/types.ts | 13 +- 9 files changed, 277 insertions(+), 61 deletions(-) diff --git a/src/analytics/MetaRouterAnalyticsClient.test.ts b/src/analytics/MetaRouterAnalyticsClient.test.ts index 5821bdd..887445f 100644 --- a/src/analytics/MetaRouterAnalyticsClient.test.ts +++ b/src/analytics/MetaRouterAnalyticsClient.test.ts @@ -681,34 +681,33 @@ describe('MetaRouterAnalyticsClient', () => { expect(stopSpy).toHaveBeenCalled(); }); - it('overflow events flush to overflow disk on memory cap exceeded', async () => { + it('overflow events flush to disk when memory cap is exceeded', async () => { const monitor = new StubNetworkMonitor('disconnected'); const client = new MetaRouterAnalyticsClient( { ...opts, - maxQueueBytes: 500, // very small cap + // Small count cap so the overflow fires quickly. + maxQueueEvents: 10, flushIntervalSeconds: 3600, }, { networkMonitor: monitor } ); await client.init(); - // Spy on flushEventsToDisk const flushSpy = jest.spyOn( (client as any).persistentQueue, 'flushEventsToDisk' ); - // Override to prevent auto-flush from interfering + // Prevent auto-flush from interfering with the capacity check. jest.spyOn(client as any, 'flush').mockResolvedValue(undefined); (client as any).dispatcher.opts.autoFlushThreshold = 9999; - // Track enough events to overflow memory queue + // Track enough events to overflow the count cap. for (let i = 0; i < 30; i++) { client.track(`event_${i}`, { data: 'x'.repeat(20) }); } - // Check that events were flushed to overflow disk expect(flushSpy).toHaveBeenCalled(); }); @@ -884,4 +883,137 @@ describe('MetaRouterAnalyticsClient', () => { ); }); }); + + describe('maxDiskEvents validation', () => { + it('rejects negative maxDiskEvents at construction', () => { + expect( + () => + new MetaRouterAnalyticsClient({ + ...opts, + maxDiskEvents: -1, + }) + ).toThrow(/maxDiskEvents.*must be >= 0/); + }); + + it('accepts maxDiskEvents: 0 (persistence disabled)', () => { + expect( + () => + new MetaRouterAnalyticsClient({ + ...opts, + maxDiskEvents: 0, + }) + ).not.toThrow(); + }); + + it('warns when 0 < maxDiskEvents < maxQueueEvents', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 2000, + maxDiskEvents: 100, + }); + expect(warnSpy).toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining( + 'maxDiskEvents (100) is less than maxQueueEvents (2000)' + ) + ); + warnSpy.mockRestore(); + }); + + it('does not warn when maxDiskEvents === maxQueueEvents', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 500, + maxDiskEvents: 500, + }); + expect(warnSpy).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('is less than maxQueueEvents') + ); + warnSpy.mockRestore(); + }); + + it('does not warn when maxDiskEvents === 0 (disabled, not inverted)', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 2000, + maxDiskEvents: 0, + }); + expect(warnSpy).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('is less than maxQueueEvents') + ); + warnSpy.mockRestore(); + }); + + it('does not warn when maxDiskEvents > maxQueueEvents', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 2000, + maxDiskEvents: 10000, + }); + expect(warnSpy).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('is less than maxQueueEvents') + ); + warnSpy.mockRestore(); + }); + }); + + describe('maxDiskEvents: 0 (persistence disabled)', () => { + it('enqueue at cap drops the oldest event (ring buffer)', async () => { + const monitor = new StubNetworkMonitor('disconnected'); + const client = new MetaRouterAnalyticsClient( + { + ...opts, + maxQueueEvents: 3, + maxDiskEvents: 0, + flushIntervalSeconds: 3600, + }, + { networkMonitor: monitor } + ); + await client.init(); + (client as any).dispatcher.opts.autoFlushThreshold = 9999; + + const flushSpy = jest.spyOn( + (client as any).persistentQueue, + 'flushEventsToDisk' + ); + + for (let i = 0; i < 5; i++) { + client.track(`event_${i}`); + } + + // With 5 enqueues and cap 3, 2 oldest should have been dropped. + const queue = (client as any).queue; + expect(queue.length).toBe(3); + // No disk write — persistence is disabled. + expect(flushSpy).not.toHaveBeenCalled(); + }); + + it('background flush does not touch disk when persistence is disabled', async () => { + const client = new MetaRouterAnalyticsClient({ + ...opts, + maxDiskEvents: 0, + }); + await client.init(); + + const pq = (client as any).persistentQueue; + const flushSpy = jest.spyOn(pq, 'flushEventsToDisk'); + + (global as any).fetch = jest.fn(() => + Promise.reject(new Error('offline')) + ); + client.track('evt1'); + client.track('evt2'); + + await pq.flushToDisk(); + + expect(flushSpy).not.toHaveBeenCalled(); + }); + }); }); diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index 03e6fc9..aadca6c 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -38,8 +38,14 @@ export class MetaRouterAnalyticsClient { private appState: AppStateStatus = AppState.currentState; private appStateSubscription: { remove?: () => void } | null = null; private identityManager: IdentityManager; - private static readonly MAX_QUEUE_SIZE = 20; - private maxQueueBytes: number = 5 * 1024 * 1024; // 5MB default + private static readonly AUTO_FLUSH_THRESHOLD = 20; + private static readonly DEFAULT_MAX_QUEUE_EVENTS = 2000; + private static readonly DEFAULT_MAX_DISK_EVENTS = 10_000; + // Byte cap is intentionally internal (parity with iOS/Android). Not a + // public option. 5MB — revisit only if a customer actually needs to tune. + private static readonly MAX_QUEUE_BYTES = 5 * 1024 * 1024; + private maxDiskEvents: number = + MetaRouterAnalyticsClient.DEFAULT_MAX_DISK_EVENTS; private dispatcher!: Dispatcher; private persistentQueue!: PersistentEventQueue; private tracingEnabled: boolean = false; @@ -83,6 +89,28 @@ export class MetaRouterAnalyticsClient { this.writeKey = writeKey; this.flushIntervalSeconds = flushIntervalSeconds ?? 10; + // Validate + normalize persistence caps. + const rawMaxDisk = + options.maxDiskEvents ?? + MetaRouterAnalyticsClient.DEFAULT_MAX_DISK_EVENTS; + if (rawMaxDisk < 0) { + throw new Error( + 'MetaRouterAnalyticsClient initialization failed: `maxDiskEvents` must be >= 0 (use 0 to disable disk persistence).' + ); + } + this.maxDiskEvents = rawMaxDisk; + + const rawMaxQueue = + options.maxQueueEvents ?? + MetaRouterAnalyticsClient.DEFAULT_MAX_QUEUE_EVENTS; + const maxQueueEvents = Math.max(1, rawMaxQueue); + + if (this.maxDiskEvents > 0 && this.maxDiskEvents < maxQueueEvents) { + warn( + `maxDiskEvents (${this.maxDiskEvents}) is less than maxQueueEvents (${maxQueueEvents}) — memory can hold more events than disk can preserve; events may be dropped during background flush` + ); + } + setDebugLogging(options.debug ?? false); this.identityManager = new IdentityManager(); // Default: wrap the raw native monitor with the asymmetric debounce @@ -91,14 +119,15 @@ export class MetaRouterAnalyticsClient { // behavior explicitly. this.networkMonitor = deps?.networkMonitor ?? new DebouncedNetworkMonitor(new NetworkMonitor()); - this.maxQueueBytes = options.maxQueueBytes ?? this.maxQueueBytes; this.dispatcher = new Dispatcher({ - maxQueueBytes: this.maxQueueBytes, - autoFlushThreshold: MetaRouterAnalyticsClient.MAX_QUEUE_SIZE, + maxEventCount: maxQueueEvents, + maxQueueBytes: MetaRouterAnalyticsClient.MAX_QUEUE_BYTES, + autoFlushThreshold: MetaRouterAnalyticsClient.AUTO_FLUSH_THRESHOLD, maxBatchSize: 100, flushIntervalSeconds: this.flushIntervalSeconds, baseRetryDelayMs: 1000, maxRetryDelayMs: 8000, + isPersistenceEnabled: () => this.maxDiskEvents > 0, isNetworkAvailable: () => this.networkStatus === 'connected', endpoint: (path) => this.endpoint(path), fetchWithTimeout: (url, init, timeoutMs) => @@ -146,7 +175,7 @@ export class MetaRouterAnalyticsClient { this.queue = this.dispatcher.getQueueRef(); this.persistentQueue = new PersistentEventQueue(this.dispatcher, { - maxDiskEvents: options.maxOfflineDiskEvents, + maxDiskEvents: this.maxDiskEvents, }); } @@ -516,7 +545,8 @@ export class MetaRouterAnalyticsClient { flushInFlight: d.flushInFlight, circuitState: d.circuitState, circuitRemainingMs: d.circuitRemainingMs, - maxQueueBytes: d.maxQueueBytes, + maxQueueEvents: d.maxQueueEvents, + maxDiskEvents: this.maxDiskEvents, tracingEnabled: this.tracingEnabled, hasDiskData: this.persistentQueue.hasDiskData, networkStatus: this.networkStatus, diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 1de5e8f..32d86de 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -2,12 +2,14 @@ import Dispatcher from './dispatcher'; import CircuitBreaker from './utils/circuitBreaker'; const baseOpts = () => ({ + maxEventCount: 2000, maxQueueBytes: 5 * 1024 * 1024, // 5MB autoFlushThreshold: 20, maxBatchSize: 100, flushIntervalSeconds: 3600, // keep timer quiet unless started baseRetryDelayMs: 1000, maxRetryDelayMs: 8000, + isPersistenceEnabled: () => true, isNetworkAvailable: () => true, endpoint: (p: string) => `https://example.com${p}`, fetchWithTimeout: jest.fn( diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index 6adb9fd..520418f 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -8,13 +8,20 @@ export interface NetworkResponse { } export interface DispatcherOptions { - maxQueueBytes: number; // byte-based cap on memory queue (~5MB default) + maxEventCount: number; // event-count cap on memory queue (default 2000) + maxQueueBytes: number; // byte-based cap on memory queue (~5MB default, internal) autoFlushThreshold: number; // e.g., 20 maxBatchSize: number; // initial max; may shrink on 413 flushIntervalSeconds: number; baseRetryDelayMs: number; // retry floor base delay (default 1000) maxRetryDelayMs: number; // retry floor cap (default 8000) + /** + * Whether disk persistence is enabled. When false, capacity overflow drops + * the oldest event (ring buffer) instead of splicing the whole queue to + * the onCapacityOverflow handler, since there is no disk to spill to. + */ + isPersistenceEnabled: () => boolean; isNetworkAvailable: () => boolean; // returns false when offline endpoint: (path: string) => string; @@ -144,25 +151,58 @@ export default class Dispatcher { return Dispatcher.encoder.encode(JSON.stringify(event)).byteLength; } - enqueue(event: EventPayload): void { - const eventSize = this.estimateEventSize(event); - - // Capacity overflow: flush entire queue to overflow disk, then reset - if ( - this.queue.length > 0 && - this.queueSizeBytes + eventSize > this.opts.maxQueueBytes - ) { - if (this.opts.onCapacityOverflow) { - const flushed = this.queue.splice(0); - this.queueSizeBytes = 0; - this.opts.onCapacityOverflow(flushed); - } else { + private handleCapacityOverflow(triggerDesc: string): void { + // Persistence disabled (maxDiskEvents === 0): ring-buffer — drop oldest + // event one at a time instead of draining the whole queue, since there + // is no disk to spill to. + if (!this.opts.isPersistenceEnabled()) { + const dropped = this.queue.shift(); + if (dropped !== undefined) { + this.queueSizeBytes -= this.estimateEventSize(dropped); + if (this.queueSizeBytes < 0) this.queueSizeBytes = 0; this.opts.warn( - `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` + `Queue cap reached (${triggerDesc}) — dropped oldest event` ); - this.queue.length = 0; - this.queueSizeBytes = 0; } + return; + } + + // Persistence enabled: splice the whole queue to the disk handler. + if (this.opts.onCapacityOverflow) { + const flushed = this.queue.splice(0); + this.queueSizeBytes = 0; + this.opts.onCapacityOverflow(flushed); + } else { + this.opts.warn( + `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` + ); + this.queue.length = 0; + this.queueSizeBytes = 0; + } + } + + private isOverCap(extraBytes: number = 0, extraCount: number = 0): boolean { + return ( + this.queue.length + extraCount > this.opts.maxEventCount || + this.queueSizeBytes + extraBytes > this.opts.maxQueueBytes + ); + } + + enqueue(event: EventPayload): void { + const eventSize = this.estimateEventSize(event); + + // Enforce cap: if adding this event would exceed count OR byte limit, + // flush/evict first so the new event still fits. + while (this.queue.length > 0 && this.isOverCap(eventSize, 1)) { + this.handleCapacityOverflow( + this.queueSizeBytes + eventSize > this.opts.maxQueueBytes + ? 'bytes' + : 'count' + ); + if (!this.opts.isPersistenceEnabled() && this.queue.length === 0) break; + // When persistence IS enabled, handleCapacityOverflow drains the whole + // queue so the loop exits naturally. + if (this.opts.isPersistenceEnabled()) break; } this.opts.log( @@ -188,19 +228,14 @@ export default class Dispatcher { this.queueSizeBytes += this.estimateEventSize(e); } - // Enforce cap: if over byte limit, flush entire queue to overflow disk - if (this.queueSizeBytes > this.opts.maxQueueBytes) { - if (this.opts.onCapacityOverflow) { - const flushed = this.queue.splice(0); - this.queueSizeBytes = 0; - this.opts.onCapacityOverflow(flushed); - } else { - this.opts.warn( - `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` - ); - this.queue.length = 0; - this.queueSizeBytes = 0; - } + // Enforce cap: drain to disk (or ring-buffer drop-oldest in 0-mode). + while (this.isOverCap()) { + const before = this.queue.length; + this.handleCapacityOverflow( + this.queueSizeBytes > this.opts.maxQueueBytes ? 'bytes' : 'count' + ); + if (this.queue.length === before) break; // safety: avoid infinite loop + if (this.opts.isPersistenceEnabled()) break; // drained entire queue } if (this.queue.length >= this.opts.autoFlushThreshold) { @@ -251,7 +286,7 @@ export default class Dispatcher { const doFlush = async () => { while (this.queue.length) { if (!this.opts.isNetworkAvailable()) { - if (this.opts.onFlushToDisk) { + if (this.opts.onFlushToDisk && this.opts.isPersistenceEnabled()) { const flushed = this.queue.splice(0); this.queueSizeBytes = 0; this.opts.onFlushToDisk(flushed); @@ -489,7 +524,7 @@ export default class Dispatcher { flushInFlight: !!this.flushInFlight, circuitState: this.circuit.getState(), circuitRemainingMs: this.circuit.remainingCooldownMs(), - maxQueueBytes: this.opts.maxQueueBytes, + maxQueueEvents: this.opts.maxEventCount, maxBatchSize: this.maxBatchSize, consecutiveRetries: this.consecutiveRetries, isNetworkAvailable: this.opts.isNetworkAvailable(), diff --git a/src/analytics/init.test.ts b/src/analytics/init.test.ts index 1df6ba2..66f5695 100644 --- a/src/analytics/init.test.ts +++ b/src/analytics/init.test.ts @@ -147,7 +147,7 @@ describe('createAnalyticsClient', () => { }); }); - it('supports reconfiguration after reset with new maxQueueBytes', async () => { + it('supports reconfiguration after reset with new maxQueueEvents', async () => { // Note: This test documents the proper pattern: // 1. await reset() before reconfiguring // 2. The warning added in createAnalyticsClient catches forgotten awaits @@ -167,42 +167,42 @@ describe('createAnalyticsClient', () => { const { createAnalyticsClient } = require('./init'); - // First client with maxQueueBytes: 3MB + // First client with maxQueueEvents: 1000 const client1 = createAnalyticsClient({ ...opts, - maxQueueBytes: 3 * 1024 * 1024, + maxQueueEvents: 1000, }); // Wait for init to complete await new Promise((resolve) => setTimeout(resolve, 100)); let debug1 = await client1.getDebugInfo(); - expect(debug1?.maxQueueBytes).toBe(3 * 1024 * 1024); + expect(debug1?.maxQueueEvents).toBe(1000); // Properly await reset before reconfiguring await client1.reset(); - // Create new client with maxQueueBytes: 5MB + // Create new client with maxQueueEvents: 3000 const client2 = createAnalyticsClient({ ...opts, - maxQueueBytes: 5 * 1024 * 1024, + maxQueueEvents: 3000, }); // Wait for new init to complete await new Promise((resolve) => setTimeout(resolve, 100)); const debug2 = await client2.getDebugInfo(); - expect(debug2?.maxQueueBytes).toBe(5 * 1024 * 1024); + expect(debug2?.maxQueueEvents).toBe(3000); }); it('warns if reconfiguration attempted without awaiting reset', async () => { const { createAnalyticsClient } = require('./init'); const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); - // First client with maxQueueBytes: 3MB + // First client with maxQueueEvents: 1000 const client1 = createAnalyticsClient({ ...opts, - maxQueueBytes: 3 * 1024 * 1024, + maxQueueEvents: 1000, }); await new Promise((resolve) => setTimeout(resolve, 50)); @@ -213,7 +213,7 @@ describe('createAnalyticsClient', () => { // Immediately try to create with new config const client2 = createAnalyticsClient({ ...opts, - maxQueueBytes: 5 * 1024 * 1024, + maxQueueEvents: 3000, }); // Should warn about config change diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index 485256b..e1120ec 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -83,8 +83,13 @@ export class PersistentEventQueue { * Flush the in-memory queue to disk (append + cap). * Called on app background and when the flush-to-disk threshold is hit. * Serialized against other disk writes. + * + * No-op when persistence is disabled (maxDiskEvents === 0). Memory queue + * is left intact so a subsequent foreground flush can still try to send + * those events; on app kill they are lost (documented tradeoff). */ async flushToDisk(): Promise { + if (this.maxDiskEvents === 0) return; const queue = this.dispatcher.getQueueRef(); if (queue.length === 0) return; const events = this.dispatcher.drainQueue(); @@ -95,9 +100,14 @@ export class PersistentEventQueue { * Append explicit events to the disk store (read-merge-cap-write). * Called by dispatcher onCapacityOverflow / onFlushToDisk callbacks. * Serialized against other disk writes. + * + * No-op when persistence is disabled (maxDiskEvents === 0) — events + * passed here have already been spliced out of the memory queue by the + * dispatcher, so in 0-mode they are lost. */ async flushEventsToDisk(events: EventPayload[]): Promise { if (events.length === 0) return; + if (this.maxDiskEvents === 0) return; const prev = this.diskWriteInFlight; const next = (async () => { diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 19815dc..d3a40fc 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -19,12 +19,14 @@ function mockNativeStorage() { function createDispatcher(overrides?: Partial) { return new Dispatcher({ + maxEventCount: 2000, maxQueueBytes: 5 * 1024 * 1024, autoFlushThreshold: 9999, maxBatchSize: 100, flushIntervalSeconds: 3600, baseRetryDelayMs: 1000, maxRetryDelayMs: 8000, + isPersistenceEnabled: () => true, isNetworkAvailable: () => true, endpoint: (p: string) => `https://example.com${p}`, fetchWithTimeout: jest.fn(async () => ({ ok: true, status: 200 }) as any), diff --git a/src/analytics/persistence/__tests__/integration.test.ts b/src/analytics/persistence/__tests__/integration.test.ts index 6da7c4b..edc9e3c 100644 --- a/src/analytics/persistence/__tests__/integration.test.ts +++ b/src/analytics/persistence/__tests__/integration.test.ts @@ -188,8 +188,8 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { { writeKey: 'test-key', ingestionHost: 'https://example.com', - maxQueueBytes: 500, - maxOfflineDiskEvents: 100, + maxQueueEvents: 10, + maxDiskEvents: 100, }, { networkMonitor: monitor } ); diff --git a/src/analytics/types.ts b/src/analytics/types.ts index dd49392..6304eb0 100644 --- a/src/analytics/types.ts +++ b/src/analytics/types.ts @@ -31,10 +31,15 @@ export interface InitOptions { ingestionHost: string; flushIntervalSeconds?: number; debug?: boolean; - /** Max bytes (UTF-8) held in memory queue; oldest are dropped once cap is hit (default: 5MB) */ - maxQueueBytes?: number; - /** Max events stored on disk during extended offline periods (default: 10000) */ - maxOfflineDiskEvents?: number; + /** Max events held in the in-memory queue (default: 2000). Clamped to >= 1. */ + maxQueueEvents?: number; + /** + * Max events stored on disk for crash-safety + extended offline periods + * (default: 10000). Must be >= 0. When set to 0, disk persistence is + * disabled and the SDK runs as a pure in-memory pipeline (events are lost + * on app kill — documented tradeoff). + */ + maxDiskEvents?: number; } export interface AnalyticsInterface { From 225e086d70f26b6db83fc074952fd6f74152daa4 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Thu, 16 Apr 2026 20:11:28 -0600 Subject: [PATCH 07/10] feat: disk drain fatal-config fires onFatalConfig handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: the disk drain path silently deleted the disk store on 401/403/404 while the main flush path disabled the client. Asymmetric — the client would stay "ready" but drop events until the next main flush also hit the same error. Now both paths converge on a shared Dispatcher.handleFatalConfig helper that clears the queue, stops the flush loop, and fires onFatalConfig. Also adds test coverage for: - drain FATAL_CONFIG triggers onFatalConfig - enqueue at the event-count cap (count-cap overflow parity with byte-cap overflow) - ring-buffer drop-oldest when persistence is disabled (maxDiskEvents=0) --- src/analytics/dispatcher.test.ts | 42 +++++++++++++++++++ src/analytics/dispatcher.ts | 19 ++++++--- .../persistence/PersistentEventQueue.ts | 4 ++ .../__tests__/PersistentEventQueue.test.ts | 24 +++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 32d86de..7b0c6fe 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -500,6 +500,48 @@ describe('Dispatcher', () => { expect(opts.onCapacityOverflow).not.toHaveBeenCalled(); }); + it('fires when the event-count cap is hit (even when well under byte cap)', () => { + const opts = baseOpts(); + const overflowEvents: any[][] = []; + opts.onCapacityOverflow = jest.fn((events: any[]) => { + overflowEvents.push(events); + }); + opts.maxEventCount = 3; + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + for (let i = 0; i < 3; i++) { + d.enqueue({ type: 'track', event: `e${i}` } as any); + } + // 4th event would exceed count cap — the existing 3 are spliced out. + d.enqueue({ type: 'track', event: 'e3' } as any); + + expect(opts.onCapacityOverflow).toHaveBeenCalledTimes(1); + expect(overflowEvents[0].length).toBe(3); + expect(d.getQueueRef().length).toBe(1); + expect((d.getQueueRef()[0] as any).event).toBe('e3'); + }); + + it('drops oldest (ring buffer) when persistence is disabled', () => { + const opts = baseOpts(); + opts.onCapacityOverflow = jest.fn(); + opts.isPersistenceEnabled = () => false; + opts.maxEventCount = 3; + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + for (let i = 0; i < 5; i++) { + d.enqueue({ type: 'track', event: `e${i}` } as any); + } + + // Queue still size 3 (ring buffer); the 2 oldest were dropped one by one. + expect(d.getQueueRef().length).toBe(3); + const events = d.getQueueRef().map((e: any) => e.event); + expect(events).toEqual(['e2', 'e3', 'e4']); + // No disk splice happened. + expect(opts.onCapacityOverflow).not.toHaveBeenCalled(); + }); + it('resets queue bytes to 0 after overflow', () => { const opts = baseOpts(); opts.onCapacityOverflow = jest.fn(); diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index 520418f..a448ec8 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -387,11 +387,7 @@ export default class Dispatcher { } if (s === 401 || s === 403 || s === 404) { - this.opts.error(`Fatal config error ${s}. Disabling client.`); - this.queue.length = 0; - this.queueSizeBytes = 0; - this.stop(); - this.opts.onFatalConfig?.(); + this.handleFatalConfig(s); return; } @@ -474,6 +470,19 @@ export default class Dispatcher { this.consecutiveRetries = 0; } + /** + * Coordinate shutdown on a fatal config error (401/403/404). + * Called by the main flush path and by the disk drain so both paths + * converge to the same "disable client" signal. + */ + handleFatalConfig(statusCode: number): void { + this.opts.error(`Fatal config error ${statusCode}. Disabling client.`); + this.queue.length = 0; + this.queueSizeBytes = 0; + this.stop(); + this.opts.onFatalConfig?.(); + } + /** * Send a batch directly to network, bypassing the memory queue. * Used by disk drain to flush overflow without loading into queue. diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index e1120ec..7d5cf7b 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -318,6 +318,10 @@ export class PersistentEventQueue { ); await nativeDeleteSnapshot(); this._hasDiskData = false; + // Coordinate shutdown with the main dispatcher. Previously the + // drain path silently swallowed 401/403/404 while the main flush + // path disabled the client — inconsistent behavior. + dispatcher.handleFatalConfig(response.statusCode); return; } diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index d3a40fc..569cb95 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -515,6 +515,30 @@ describe('PersistentEventQueue', () => { expect(store.data).toBeNull(); }); + it('fires onFatalConfig handler on 401/403/404 during drain', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); + + const onFatalConfig = jest.fn(); + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 401, + })), + onFatalConfig, + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(onFatalConfig).toHaveBeenCalledTimes(1); + }); + it('drops batch on other 4xx and continues draining', async () => { const { store } = mockNativeStorage(); From f671ceb2efd139e05598a836612f3a7a08ad966f Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Thu, 16 Apr 2026 20:13:22 -0600 Subject: [PATCH 08/10] chore: align network status logs with canonical phrasing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matches the iOS/Android wording from the port guide so support can grep the same strings across all platforms: - "Network status changed: -> " - "Dispatcher paused — device is offline" - "Dispatcher resumed — device is online, triggering flush" No behavior change. networkStatus + hasDiskData are already in getDebugInfo from earlier commits; log set already omits the "deleted during iteration" noise (reconcile, per-update, debounce-state). --- src/analytics/MetaRouterAnalyticsClient.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index aadca6c..f545021 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -230,15 +230,18 @@ export class MetaRouterAnalyticsClient { const wasOffline = this.networkStatus === 'disconnected'; this.networkStatus = status; + log( + `Network status changed: ${wasOffline ? 'disconnected' : 'connected'} -> ${status}` + ); if (wasOffline && status === 'connected') { - log('Network connectivity restored — resuming flush'); + log('Dispatcher resumed — device is online, triggering flush'); this.dispatcher.resetCircuitBreaker(); // (1) Memory queue → network (existing path) void this.flush(); // (2) Disk → network directly (no load into memory queue) void this.persistentQueue.drainDiskToNetwork(this.dispatcher); } else if (status === 'disconnected') { - log('Network connectivity lost — pausing HTTP attempts'); + log('Dispatcher paused — device is offline'); } } ); From 3b208db99a1b274c743da7412612d2bbb8538220 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Fri, 17 Apr 2026 16:39:25 -0600 Subject: [PATCH 09/10] fix: dispatcher logic --- src/analytics/dispatcher.test.ts | 102 +++++++++++++++++++----- src/analytics/dispatcher.ts | 6 +- src/analytics/utils/responseCategory.ts | 2 +- 3 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 7b0c6fe..89bbca1 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -187,31 +187,42 @@ describe('Dispatcher', () => { }); it('gradually recovers maxBatchSize after 413-induced reduction on subsequent successes', async () => { - const opts = baseOpts(); - opts.maxBatchSize = 100; - opts.autoFlushThreshold = 9999; - let callCount = 0; - opts.fetchWithTimeout = jest.fn(async (_url?: string, _init?: any) => { - callCount++; - // First call: 413 to trigger halving - if (callCount === 1) - return { ok: false, status: 413, headers: { get: () => null } } as any; - return { ok: true, status: 200 } as any; - }); + jest.useFakeTimers(); + try { + const opts = baseOpts(); + opts.maxBatchSize = 100; + opts.autoFlushThreshold = 9999; + let callCount = 0; + opts.fetchWithTimeout = jest.fn(async (_url?: string, _init?: any) => { + callCount++; + // First call: 413 to trigger halving + if (callCount === 1) + return { + ok: false, + status: 413, + headers: { get: () => null }, + } as any; + return { ok: true, status: 200 } as any; + }); - const d = new Dispatcher(opts); + const d = new Dispatcher(opts); - // Enqueue enough events and flush to trigger the 413 - for (let i = 0; i < 5; i++) - d.enqueue({ type: 'track', event: `e${i}` } as any); - await d.flush(); + // Enqueue enough events and flush to trigger the 413 + for (let i = 0; i < 5; i++) + d.enqueue({ type: 'track', event: `e${i}` } as any); + await d.flush(); - // After 413, maxBatchSize should have been halved to 50 - expect(d.getDebugInfo().maxBatchSize).toBe(50); + // After 413, maxBatchSize should have been halved to 50 + expect(d.getDebugInfo().maxBatchSize).toBe(50); - // Flush again — events succeed, so batch size should recover: min(50*2, 100) = 100 - await d.flush(); - expect(d.getDebugInfo().maxBatchSize).toBe(100); + // Advance past the scheduled retry (500ms) — the internal retry fires + // success, which recovers batch size: min(50*2, 100) = 100. + await jest.advanceTimersByTimeAsync(500); + + expect(d.getDebugInfo().maxBatchSize).toBe(100); + } finally { + jest.useRealTimers(); + } }); it('does not recover maxBatchSize beyond initialMaxBatchSize', async () => { @@ -397,6 +408,9 @@ describe('Dispatcher', () => { it('circuit breaker does NOT reset while still connected but failing', async () => { const opts = baseOpts(); + // Drop chunks on failure (don't schedule retry) so each manual flush + // can run and the circuit sees all three failures. + opts.isOperational = () => false; opts.fetchWithTimeout = jest.fn( async () => ({ ok: false, status: 500, headers: { get: () => null } }) as any @@ -630,6 +644,52 @@ describe('Dispatcher', () => { expect(opts.onFlushComplete).not.toHaveBeenCalled(); }); + + it('does not fire when all batches are dropped as 4xx (no 2xx observed)', async () => { + const opts = baseOpts(); + opts.onFlushComplete = jest.fn(); + opts.autoFlushThreshold = 9999; + opts.fetchWithTimeout = jest.fn( + async () => + ({ ok: false, status: 400, headers: { get: () => null } }) as any + ); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + await d.flush(); + + expect(opts.onFlushComplete).not.toHaveBeenCalled(); + expect(d.getQueueRef().length).toBe(0); + }); + }); + + describe('retry-backoff churn guard', () => { + it('short-circuits flush while a retry timer is armed', async () => { + const opts = baseOpts(); + opts.autoFlushThreshold = 9999; + opts.fetchWithTimeout = jest.fn( + async () => + ({ ok: false, status: 500, headers: { get: () => null } }) as any + ); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + await d.flush(); + + const fetchSpy = opts.fetchWithTimeout as jest.Mock; + const callsAfterRetryArmed = fetchSpy.mock.calls.length; + expect(opts.onScheduleFlushIn).toHaveBeenCalled(); + + // External callers (periodic timer, enqueue auto-flush, onFlushComplete) + // must not bypass the backoff window. + await d.flush(); + await d.flush(); + await d.flush(); + + expect(fetchSpy.mock.calls.length).toBe(callsAfterRetryArmed); + }); }); describe('sendBatchDirect', () => { diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index a448ec8..53f816c 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -281,6 +281,10 @@ export default class Dispatcher { async flush(): Promise { if (!this.queue.length) return; if (this.flushInFlight) return this.flushInFlight; + // A retry is armed — let it fire on its own schedule instead of + // launching immediately and bypassing the backoff window. The scheduled + // callback nils nextTimer before calling flush(), so it passes through. + if (this.nextTimer) return; if (!this.opts.canSend()) return; const doFlush = async () => { @@ -485,7 +489,7 @@ export default class Dispatcher { /** * Send a batch directly to network, bypassing the memory queue. - * Used by disk drain to flush overflow without loading into queue. + * Used by the disk drain to send events without loading them into the queue. * Returns { statusCode } on HTTP response, null on network/transport error. */ async sendBatchDirect( diff --git a/src/analytics/utils/responseCategory.ts b/src/analytics/utils/responseCategory.ts index c78a9e0..927b625 100644 --- a/src/analytics/utils/responseCategory.ts +++ b/src/analytics/utils/responseCategory.ts @@ -1,6 +1,6 @@ /** * Shared classification of HTTP status codes used by both - * the dispatcher and the overflow disk drain. + * the dispatcher and the disk drain. */ export enum ResponseCategory { SUCCESS = 'SUCCESS', From 81706d43a175886fbc6c9ebf93afef5894add361 Mon Sep 17 00:00:00 2001 From: Christopher Houdlette Date: Wed, 22 Apr 2026 12:05:50 -0600 Subject: [PATCH 10/10] fix: improving event durability --- jest.setup.js | 1 + .../MetaRouterAnalyticsClient.test.ts | 6 +- src/analytics/MetaRouterAnalyticsClient.ts | 18 ++- src/analytics/dispatcher.test.ts | 40 ++++++ src/analytics/dispatcher.ts | 48 +++++-- .../persistence/NativeQueueStorage.ts | 11 +- .../persistence/PersistentEventQueue.ts | 131 +++++++++++++++--- .../__tests__/NativeQueueStorage.test.ts | 33 +++-- .../__tests__/PersistentEventQueue.test.ts | 85 +++++++++++- 9 files changed, 321 insertions(+), 52 deletions(-) diff --git a/jest.setup.js b/jest.setup.js index 0e49fc6..f921127 100644 --- a/jest.setup.js +++ b/jest.setup.js @@ -13,6 +13,7 @@ jest.mock('react-native', () => ({ }, NativeModules: { MetaRouterQueueStorage: { + exists: jest.fn(() => Promise.resolve(false)), readSnapshot: jest.fn(() => Promise.resolve(null)), writeSnapshot: jest.fn(() => Promise.resolve()), deleteSnapshot: jest.fn(() => Promise.resolve()), diff --git a/src/analytics/MetaRouterAnalyticsClient.test.ts b/src/analytics/MetaRouterAnalyticsClient.test.ts index 2146582..12b825b 100644 --- a/src/analytics/MetaRouterAnalyticsClient.test.ts +++ b/src/analytics/MetaRouterAnalyticsClient.test.ts @@ -266,10 +266,10 @@ describe('MetaRouterAnalyticsClient', () => { (client as any).dispatcher && ((client as any).dispatcher.opts.autoFlushThreshold = 9999); - // Spy on overflow to verify events get flushed + // Spy on overflow to verify events are handed to the disk buffer const flushSpy = jest.spyOn( (client as any).persistentQueue, - 'flushEventsToDisk' + 'bufferEventsForDisk' ); // Track one event to measure enriched size, then set byte cap for 10 events @@ -745,7 +745,7 @@ describe('MetaRouterAnalyticsClient', () => { const flushSpy = jest.spyOn( (client as any).persistentQueue, - 'flushEventsToDisk' + 'bufferEventsForDisk' ); // Prevent auto-flush from interfering with the capacity check. diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index b87dedd..7dc57d7 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -157,9 +157,8 @@ export class MetaRouterAnalyticsClient { warn, error, onCapacityOverflow: (events) => - void this.persistentQueue.flushEventsToDisk(events), - onFlushToDisk: (events) => - void this.persistentQueue.flushEventsToDisk(events), + this.persistentQueue.bufferEventsForDisk(events), + onFlushToDisk: (events) => this.persistentQueue.flushEventsToDisk(events), onFlushComplete: () => { if (this.networkStatus === 'connected') { void this.persistentQueue.drainDiskToNetwork(this.dispatcher); @@ -332,7 +331,9 @@ export class MetaRouterAnalyticsClient { // Check if we should flush to disk based on thresholds if (this.persistentQueue.shouldFlushToDisk()) { - void this.persistentQueue.flushToDisk(); + void this.persistentQueue.flushToDisk().catch((err) => { + warn('Failed to flush queue to disk after threshold:', err); + }); } } @@ -368,8 +369,13 @@ export class MetaRouterAnalyticsClient { log('App moved to background'); this.stopFlushLoop(); this.clearNextTimer(); - await this.flush(); - await this.persistentQueue.flushToDisk(); + try { + await this.flush(); + await this.persistentQueue.flushToDisk(); + await this.persistentQueue.flushPendingDiskWrites(); + } catch (err) { + warn('Failed to persist queue while moving to background:', err); + } } if (nextState === 'active' && this.lifecycle === 'ready') { log('App moved to foreground'); diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 89bbca1..d3c464e 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -454,6 +454,27 @@ describe('Dispatcher', () => { ).toBe(true); }); + it('restores events to memory if offline disk flush fails', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.onFlushToDisk = jest.fn(async () => { + throw new Error('disk full'); + }); + opts.autoFlushThreshold = 9999; + const d = new Dispatcher(opts); + + d.enqueue({ type: 'track', event: 'e1' } as any); + d.enqueue({ type: 'track', event: 'e2' } as any); + await d.flush(); + + expect(d.getQueueRef().map((e: any) => e.event)).toEqual(['e1', 'e2']); + expect( + (opts.warn as jest.Mock).mock.calls.some((c) => + String(c[0]).includes('restored to memory') + ) + ).toBe(true); + }); + it('offline log warns with queue count when no onFlushToDisk', async () => { const opts = baseOpts(); opts.isNetworkAvailable = () => false; @@ -611,6 +632,25 @@ describe('Dispatcher', () => { expect(opts.onFlushToDisk).not.toHaveBeenCalled(); }); + + it('awaits onFlushToDisk before resolving offline flush', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.autoFlushThreshold = 9999; + let persisted = false; + opts.onFlushToDisk = jest.fn(async () => { + await Promise.resolve(); + persisted = true; + }); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + + await d.flush(); + + expect(persisted).toBe(true); + expect(d.getQueueRef().length).toBe(0); + }); }); describe('onFlushComplete callback', () => { diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index 53f816c..8e5576e 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -39,8 +39,8 @@ export interface DispatcherOptions { warn: (...args: any[]) => void; error: (...args: any[]) => void; - onCapacityOverflow?: (events: EventPayload[]) => void; // called with entire queue when byte cap is hit — flushes to disk - onFlushToDisk?: (events: EventPayload[]) => void; // called when flush triggers while offline — persists queue to disk + onCapacityOverflow?: (events: EventPayload[]) => void | Promise; // called with entire queue when cap is hit + onFlushToDisk?: (events: EventPayload[]) => void | Promise; // called when flush triggers while offline — persists queue to disk onFlushComplete?: () => void; // fires after successful online flush to trigger disk drain onScheduleFlushIn?: (ms: number) => void; // optional notification for tests/metrics onFatalConfig?: () => void; // 401/403/404 handler @@ -81,6 +81,14 @@ export default class Dispatcher { return events; } + /** + * Restore events to the front without applying capacity overflow policy. + * Used only after a failed durability handoff so events are not lost. + */ + restoreQueueFront(events: EventPayload[]): void { + this.requeueChunk(events); + } + start(): void { if (this.flushTimer) clearInterval(this.flushTimer); this.flushTimer = setInterval( @@ -171,7 +179,22 @@ export default class Dispatcher { if (this.opts.onCapacityOverflow) { const flushed = this.queue.splice(0); this.queueSizeBytes = 0; - this.opts.onCapacityOverflow(flushed); + try { + const result = this.opts.onCapacityOverflow(flushed); + void Promise.resolve(result).catch((err) => { + this.requeueChunk(flushed); + this.opts.warn( + 'Capacity overflow persistence failed — restored events to memory', + err + ); + }); + } catch (err) { + this.requeueChunk(flushed); + this.opts.warn( + 'Capacity overflow persistence failed — restored events to memory', + err + ); + } } else { this.opts.warn( `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` @@ -291,12 +314,19 @@ export default class Dispatcher { while (this.queue.length) { if (!this.opts.isNetworkAvailable()) { if (this.opts.onFlushToDisk && this.opts.isPersistenceEnabled()) { - const flushed = this.queue.splice(0); - this.queueSizeBytes = 0; - this.opts.onFlushToDisk(flushed); - this.opts.warn( - `Offline — flushed ${flushed.length} event(s) to disk` - ); + const flushed = this.drainQueue(); + try { + await this.opts.onFlushToDisk(flushed); + this.opts.warn( + `Offline — flushed ${flushed.length} event(s) to disk` + ); + } catch (err) { + this.requeueChunk(flushed); + this.opts.warn( + `Offline — failed to flush ${flushed.length} event(s) to disk; restored to memory`, + err + ); + } } else { this.opts.warn( `Offline — pausing HTTP attempts, ${this.queue.length} event(s) queued` diff --git a/src/analytics/persistence/NativeQueueStorage.ts b/src/analytics/persistence/NativeQueueStorage.ts index 01f21d5..3772d17 100644 --- a/src/analytics/persistence/NativeQueueStorage.ts +++ b/src/analytics/persistence/NativeQueueStorage.ts @@ -8,6 +8,7 @@ import { warn } from '../utils/logger'; * - readSnapshot(): Promise — full contents, or null if no file * - writeSnapshot(data: string): Promise — atomic overwrite * - deleteSnapshot(): Promise — delete if present + * - exists(): Promise — cheap existence check * * Append / merge / cap logic lives in JS (PersistentEventQueue) rather than * native so the policy stays in one place and is easy to test. @@ -50,17 +51,22 @@ export async function readSnapshot(): Promise { return await mod.readSnapshot(); } catch (err) { warn('Failed to read queue snapshot from disk:', err); - return null; + throw err; } } export async function writeSnapshot(data: string): Promise { const mod = getModule(); - if (!mod) return; + if (!mod) { + throw new Error( + 'MetaRouterQueueStorage native module is not available; cannot write queue snapshot.' + ); + } try { await mod.writeSnapshot(data); } catch (err) { warn('Failed to write queue snapshot to disk:', err); + throw err; } } @@ -71,5 +77,6 @@ export async function deleteSnapshot(): Promise { await mod.deleteSnapshot(); } catch (err) { warn('Failed to delete queue snapshot from disk:', err); + throw err; } } diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index 7d5cf7b..6b5aebb 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -30,9 +30,8 @@ const DRAIN_BATCH_SIZE = 100; * Memory + disk coordination for the event queue. * * Single disk file (queue.v1.json) backs both crash-safety (background flush) - * and offline overflow. All writes go through {@link flushEventsToDisk} which - * reads existing disk contents, merges the new events, enforces the disk cap, - * and atomically writes back. + * and offline overflow. All native file mutations run through one serialized + * lane so append, drain, and delete operations cannot interleave. * * On boot, {@link checkForPersistedEvents} does a cheap existence check and * sets {@link hasDiskData}. The in-memory queue starts empty; when online, @@ -44,8 +43,15 @@ export class PersistentEventQueue { private readonly maxDiskEvents: number; private _hasDiskData: boolean = false; - /** Serializes all disk writes (flushEventsToDisk, flushToDisk). */ - private diskWriteInFlight: Promise | null = null; + /** Serializes every native file mutation: append, drain rewrite/delete, reset delete. */ + private diskMutation: Promise = Promise.resolve(); + /** + * Capacity overflow happens from a synchronous enqueue path. These events + * are owned here until a native write succeeds, so a failed write does not + * silently discard them from JS memory. + */ + private pendingDiskEvents: EventPayload[] = []; + private pendingFlushInFlight: Promise | null = null; /** Guards drainDiskToNetwork (read-then-delete semantics). */ private drainInFlight: Promise | null = null; @@ -54,6 +60,19 @@ export class PersistentEventQueue { this.maxDiskEvents = opts?.maxDiskEvents ?? DEFAULT_MAX_DISK_EVENTS; } + private enqueueDiskMutation(fn: () => Promise): Promise { + const run = this.diskMutation.then(fn, fn); + this.diskMutation = run.then( + () => undefined, + () => undefined + ); + return run; + } + + private waitForDiskIdle(): Promise { + return this.diskMutation; + } + /** True when a disk snapshot existed at boot or was written since. */ get hasDiskData(): boolean { return this._hasDiskData; @@ -93,31 +112,78 @@ export class PersistentEventQueue { const queue = this.dispatcher.getQueueRef(); if (queue.length === 0) return; const events = this.dispatcher.drainQueue(); - return this.flushEventsToDisk(events); + try { + await this.flushEventsToDisk(events); + } catch (err) { + this.dispatcher.restoreQueueFront(events); + warn('Failed to flush queue to disk — restored events to memory:', err); + throw err; + } } /** * Append explicit events to the disk store (read-merge-cap-write). - * Called by dispatcher onCapacityOverflow / onFlushToDisk callbacks. - * Serialized against other disk writes. + * Called by lifecycle/offline paths that can await disk durability. + * Serialized against every other disk mutation. * - * No-op when persistence is disabled (maxDiskEvents === 0) — events - * passed here have already been spliced out of the memory queue by the - * dispatcher, so in 0-mode they are lost. + * No-op when persistence is disabled (maxDiskEvents === 0). */ async flushEventsToDisk(events: EventPayload[]): Promise { if (events.length === 0) return; if (this.maxDiskEvents === 0) return; - const prev = this.diskWriteInFlight; - const next = (async () => { - if (prev) await prev; - await this._doFlushEventsToDisk(events); - })(); - this.diskWriteInFlight = next.finally(() => { - if (this.diskWriteInFlight === next) this.diskWriteInFlight = null; + await this.flushPendingDiskWrites(); + return this.enqueueDiskMutation(() => this._doFlushEventsToDisk(events)); + } + + /** + * Own events from synchronous overflow paths until they are durable. + * The write is started immediately but errors are retained in the pending + * buffer for a later retry instead of being dropped. + */ + bufferEventsForDisk(events: EventPayload[]): void { + if (events.length === 0 || this.maxDiskEvents === 0) return; + this.appendPendingDiskEvents(events); + void this.flushPendingDiskWrites().catch((err) => { + warn('Failed to flush pending disk events:', err); + }); + } + + async flushPendingDiskWrites(): Promise { + if (this.maxDiskEvents === 0) { + this.pendingDiskEvents = []; + return; + } + if (this.pendingFlushInFlight) return this.pendingFlushInFlight; + + this.pendingFlushInFlight = this._flushPendingDiskWrites().finally(() => { + this.pendingFlushInFlight = null; }); - return this.diskWriteInFlight; + return this.pendingFlushInFlight; + } + + private async _flushPendingDiskWrites(): Promise { + while (this.pendingDiskEvents.length > 0) { + const events = this.pendingDiskEvents; + this.pendingDiskEvents = []; + try { + await this.enqueueDiskMutation(() => this._doFlushEventsToDisk(events)); + } catch (err) { + this.pendingDiskEvents = events.concat(this.pendingDiskEvents); + throw err; + } + } + } + + private appendPendingDiskEvents(events: EventPayload[]): void { + this.pendingDiskEvents.push(...events); + if (this.pendingDiskEvents.length > this.maxDiskEvents) { + const dropCount = this.pendingDiskEvents.length - this.maxDiskEvents; + this.pendingDiskEvents = this.pendingDiskEvents.slice(dropCount); + warn( + `Pending disk buffer cap reached — dropped ${dropCount} oldest events` + ); + } } private async _doFlushEventsToDisk(events: EventPayload[]): Promise { @@ -148,6 +214,7 @@ export class PersistentEventQueue { ); } catch (err) { warn('Failed to flush events to disk:', err); + throw err; } } @@ -176,7 +243,16 @@ export class PersistentEventQueue { async drainDiskToNetwork(dispatcher: Dispatcher): Promise { if (this.drainInFlight) return this.drainInFlight; - this.drainInFlight = this._doDrainDiskToNetwork(dispatcher).finally(() => { + this.drainInFlight = (async () => { + try { + await this.flushPendingDiskWrites(); + } catch (err) { + warn('Disk drain continuing after pending write failure:', err); + } + await this.enqueueDiskMutation(() => + this._doDrainDiskToNetwork(dispatcher) + ); + })().finally(() => { this.drainInFlight = null; }); return this.drainInFlight; @@ -353,8 +429,19 @@ export class PersistentEventQueue { * Delete the disk store (used during reset). */ async deleteSnapshot(): Promise { - await nativeDeleteSnapshot(); - this._hasDiskData = false; + if (this.pendingFlushInFlight) { + try { + await this.pendingFlushInFlight; + } catch (err) { + warn('Reset continuing after pending disk write failure:', err); + } + } + await this.waitForDiskIdle(); + this.pendingDiskEvents = []; + await this.enqueueDiskMutation(async () => { + await nativeDeleteSnapshot(); + this._hasDiskData = false; + }); } private async _readExistingEvents(): Promise { diff --git a/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts b/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts index 24ae688..9ea25c8 100644 --- a/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts +++ b/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts @@ -1,5 +1,6 @@ import { NativeModules } from 'react-native'; import { + exists, readSnapshot, writeSnapshot, deleteSnapshot, @@ -7,6 +8,7 @@ import { function mockNativeStorage() { NativeModules.MetaRouterQueueStorage = { + exists: jest.fn().mockResolvedValue(false), readSnapshot: jest.fn().mockResolvedValue(null), writeSnapshot: jest.fn().mockResolvedValue(undefined), deleteSnapshot: jest.fn().mockResolvedValue(undefined), @@ -24,6 +26,14 @@ describe('NativeQueueStorage', () => { NativeModules.MetaRouterQueueStorage = undefined as any; }); + it('exists delegates to native module', async () => { + const mock = mockNativeStorage(); + mock.exists.mockResolvedValue(true); + const result = await exists(); + expect(result).toBe(true); + expect(mock.exists).toHaveBeenCalled(); + }); + it('readSnapshot returns null when native module returns null', async () => { const mock = mockNativeStorage(); mock.readSnapshot.mockResolvedValue(null); @@ -60,10 +70,11 @@ describe('NativeQueueStorage', () => { expect(result).toBeNull(); }); - it('writeSnapshot is a no-op if native module is missing', async () => { + it('writeSnapshot rejects if native module is missing', async () => { NativeModules.MetaRouterQueueStorage = undefined as any; - // Should not throw - await writeSnapshot('data'); + await expect(writeSnapshot('data')).rejects.toThrow( + /native module is not available/ + ); }); it('deleteSnapshot is a no-op if native module is missing', async () => { @@ -72,17 +83,21 @@ describe('NativeQueueStorage', () => { await deleteSnapshot(); }); - it('readSnapshot returns null on native module error', async () => { + it('readSnapshot rejects on native module error', async () => { const mock = mockNativeStorage(); mock.readSnapshot.mockRejectedValue(new Error('disk error')); - const result = await readSnapshot(); - expect(result).toBeNull(); + await expect(readSnapshot()).rejects.toThrow('disk error'); }); - it('writeSnapshot swallows native module error', async () => { + it('writeSnapshot rejects on native module error', async () => { const mock = mockNativeStorage(); mock.writeSnapshot.mockRejectedValue(new Error('disk error')); - // Should not throw - await writeSnapshot('data'); + await expect(writeSnapshot('data')).rejects.toThrow('disk error'); + }); + + it('deleteSnapshot rejects on native module error', async () => { + const mock = mockNativeStorage(); + mock.deleteSnapshot.mockRejectedValue(new Error('disk error')); + await expect(deleteSnapshot()).rejects.toThrow('disk error'); }); }); diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 569cb95..1470005 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -185,6 +185,25 @@ describe('PersistentEventQueue', () => { expect(written.events[0].event).toBe('disk1'); expect(written.events[1].event).toBe('mem1'); }); + + it('restores memory events and rejects when native write fails', async () => { + const { mock } = mockNativeStorage(); + mock.writeSnapshot.mockRejectedValueOnce(new Error('disk full')); + + const dispatcher = createDispatcher(); + dispatcher.enqueue({ type: 'track', event: 'mem1' } as any); + dispatcher.enqueue({ type: 'track', event: 'mem2' } as any); + + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await expect(pq.flushToDisk()).rejects.toThrow('disk full'); + expect(dispatcher.getQueueRef().map((e: any) => e.event)).toEqual([ + 'mem1', + 'mem2', + ]); + expect(pq.hasDiskData).toBe(false); + }); }); describe('shouldFlushToDisk', () => { @@ -325,7 +344,7 @@ describe('PersistentEventQueue', () => { }); it('serializes concurrent writes (no races)', async () => { - const { mock } = mockNativeStorage(); + const { mock, store } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); @@ -337,6 +356,44 @@ describe('PersistentEventQueue', () => { await Promise.all([p1, p2]); expect(mock.writeSnapshot).toHaveBeenCalledTimes(2); + const written = JSON.parse(store.data!); + expect(written.events.map((e: any) => e.event)).toEqual(['a', 'b']); + }); + + it('rejects when native write fails', async () => { + const { mock } = mockNativeStorage(); + mock.writeSnapshot.mockRejectedValueOnce(new Error('disk full')); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await expect( + pq.flushEventsToDisk([{ type: 'track', event: 'a' }]) + ).rejects.toThrow('disk full'); + expect(pq.hasDiskData).toBe(false); + }); + + it('keeps buffered overflow events pending when write fails and retries later', async () => { + const { mock, store } = mockNativeStorage(); + mock.writeSnapshot + .mockRejectedValueOnce(new Error('disk full')) + .mockImplementation(async (data: string) => { + store.data = data; + }); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.bufferEventsForDisk([{ type: 'track', event: 'overflowed' }]); + + await expect(pq.flushPendingDiskWrites()).rejects.toThrow('disk full'); + expect(store.data).toBeNull(); + + await pq.flushPendingDiskWrites(); + const written = JSON.parse(store.data!); + expect(written.events.map((e: any) => e.event)).toEqual(['overflowed']); }); }); @@ -362,6 +419,32 @@ describe('PersistentEventQueue', () => { expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalled(); }); + it('flushes pending overflow writes before draining disk to network', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'disk1' }], + }); + + const sentEvents: string[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + const body = JSON.parse(init.body); + sentEvents.push(...body.batch.map((e: any) => e.event)); + return { ok: true, status: 200 } as any; + }), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + pq.bufferEventsForDisk([{ type: 'track', event: 'overflow1' }]); + await pq.drainDiskToNetwork(dispatcher); + + expect(sentEvents).toEqual(['disk1', 'overflow1']); + expect(store.data).toBeNull(); + }); + it('deletes disk file after drain and clears hasDiskData', async () => { const { store, mock } = mockNativeStorage();