diff --git a/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt b/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt index d92eabd..080cfb9 100644 --- a/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt +++ b/android/src/main/java/com/metarouter/reactnative/MetaRouterQueueStorageModule.kt @@ -40,6 +40,20 @@ class MetaRouterQueueStorageModule( } } + /** + * Cheap existence check — does not read the file contents. + * Used on boot so a large snapshot doesn't get fully parsed just to + * discover whether there's anything to drain. + */ + @ReactMethod + fun exists(promise: Promise) { + try { + promise.resolve(snapshotFile().exists()) + } catch (e: Exception) { + promise.reject("EXISTS_ERROR", "Failed to check snapshot existence", e) + } + } + @ReactMethod fun readSnapshot(promise: Promise) { try { diff --git a/ios/MetaRouterQueueStorage.m b/ios/MetaRouterQueueStorage.m index f65bf76..4c9e48a 100644 --- a/ios/MetaRouterQueueStorage.m +++ b/ios/MetaRouterQueueStorage.m @@ -48,6 +48,19 @@ - (BOOL)ensureDirectoryExists:(NSError **)error { error:error]; } +/** + * Cheap existence check — does not read the file contents. + * Used on boot so a large snapshot doesn't get fully parsed just to + * discover whether there's anything to drain. + */ +RCT_EXPORT_METHOD(exists:(RCTPromiseResolveBlock)resolve + rejecter:(RCTPromiseRejectBlock)reject) +{ + NSString *path = [self snapshotPath]; + NSFileManager *fm = [NSFileManager defaultManager]; + resolve(@([fm fileExistsAtPath:path])); +} + RCT_EXPORT_METHOD(readSnapshot:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { diff --git a/jest.setup.js b/jest.setup.js index 0e49fc6..f921127 100644 --- a/jest.setup.js +++ b/jest.setup.js @@ -13,6 +13,7 @@ jest.mock('react-native', () => ({ }, NativeModules: { MetaRouterQueueStorage: { + exists: jest.fn(() => Promise.resolve(false)), readSnapshot: jest.fn(() => Promise.resolve(null)), writeSnapshot: jest.fn(() => Promise.resolve()), deleteSnapshot: jest.fn(() => Promise.resolve()), diff --git a/src/analytics/MetaRouterAnalyticsClient.test.ts b/src/analytics/MetaRouterAnalyticsClient.test.ts index 509a12e..12b825b 100644 --- a/src/analytics/MetaRouterAnalyticsClient.test.ts +++ b/src/analytics/MetaRouterAnalyticsClient.test.ts @@ -266,29 +266,30 @@ describe('MetaRouterAnalyticsClient', () => { (client as any).dispatcher && ((client as any).dispatcher.opts.autoFlushThreshold = 9999); + // Spy on overflow to verify events are handed to the disk buffer + const flushSpy = jest.spyOn( + (client as any).persistentQueue, + 'bufferEventsForDisk' + ); + // Track one event to measure enriched size, then set byte cap for 10 events client.track('e00'); const enrichedSize = JSON.stringify((client as any).queue[0]).length; (client as any).dispatcher.opts.maxQueueBytes = enrichedSize * 10; - const firstId = (client as any).queue[0].messageId; - - // Add 15 more (padded names for uniform size) => total 16, cap ~10 => drop 6 oldest + // Add 15 more (padded names for uniform size) => total 16, cap ~10 => overflow triggered for (let i = 1; i <= 15; i++) { client.track(`e${String(i).padStart(2, '0')}`); } - // Queue should be capped - expect((client as any).queue.length).toBe(10); + // Queue should not exceed cap (entire queue flushed on overflow, only latest events remain) + expect((client as any).queue.length).toBeLessThanOrEqual(10); - // The very first event should have been dropped - expect( - (client as any).queue.find((e: any) => e.messageId === firstId) - ).toBeUndefined(); + // Overflow should have been triggered + expect(flushSpy).toHaveBeenCalled(); - // Ordering should be preserved for the tail: remaining should be e06..e15 + // The most recent event should always be present const names = (client as any).queue.map((e: any) => e.event); - expect(names[0]).toBe('e06'); expect(names[names.length - 1]).toBe('e15'); // No network calls needed for this test @@ -682,13 +683,22 @@ describe('MetaRouterAnalyticsClient', () => { expect(fetch).toHaveBeenCalled(); }); - it('offline -> online transition resets circuit breaker and triggers flush', async () => { + it('offline -> online transition resets circuit breaker and triggers flush + drain', async () => { const monitor = new StubNetworkMonitor('connected'); const client = new MetaRouterAnalyticsClient(opts, { networkMonitor: monitor, }); await client.init(); + // Spy on flushEventsToDisk and drainDiskToNetwork + const overflowSpy = jest.spyOn( + (client as any).persistentQueue, + 'flushEventsToDisk' + ); + const drainSpy = jest + .spyOn((client as any).persistentQueue, 'drainDiskToNetwork') + .mockResolvedValue(undefined); + // Go offline monitor.simulate('disconnected'); @@ -696,20 +706,16 @@ describe('MetaRouterAnalyticsClient', () => { client.track('Offline Event 1'); client.track('Offline Event 2'); - // Flush should not send (offline) + // Flush while offline — events should be moved to overflow storage await client.flush(); - // Events should still be in queue since network is unavailable - // (the dispatcher guard prevents HTTP calls) + expect(overflowSpy).toHaveBeenCalled(); - // Go online — should trigger flush - (global as any).fetch = jest - .fn() - .mockResolvedValue({ ok: true, status: 200 }); + // Go online — should trigger flush and drain monitor.simulate('connected'); - // Allow the async flush to complete + // Allow the async operations to complete await new Promise((resolve) => setTimeout(resolve, 50)); - expect(fetch).toHaveBeenCalled(); + expect(drainSpy).toHaveBeenCalled(); }); it('reset() cleans up network monitor', async () => { @@ -723,6 +729,55 @@ describe('MetaRouterAnalyticsClient', () => { await client.reset(); expect(stopSpy).toHaveBeenCalled(); }); + + it('overflow events flush to disk when memory cap is exceeded', async () => { + const monitor = new StubNetworkMonitor('disconnected'); + const client = new MetaRouterAnalyticsClient( + { + ...opts, + // Small count cap so the overflow fires quickly. + maxQueueEvents: 10, + flushIntervalSeconds: 3600, + }, + { networkMonitor: monitor } + ); + await client.init(); + + const flushSpy = jest.spyOn( + (client as any).persistentQueue, + 'bufferEventsForDisk' + ); + + // Prevent auto-flush from interfering with the capacity check. + jest.spyOn(client as any, 'flush').mockResolvedValue(undefined); + (client as any).dispatcher.opts.autoFlushThreshold = 9999; + + // Track enough events to overflow the count cap. + for (let i = 0; i < 30; i++) { + client.track(`event_${i}`, { data: 'x'.repeat(20) }); + } + + expect(flushSpy).toHaveBeenCalled(); + }); + + it('offline -> online drains disk overflow', async () => { + const monitor = new StubNetworkMonitor('connected'); + const client = new MetaRouterAnalyticsClient(opts, { + networkMonitor: monitor, + }); + await client.init(); + + // Spy on drainDiskToNetwork + const drainSpy = jest + .spyOn((client as any).persistentQueue, 'drainDiskToNetwork') + .mockResolvedValue(undefined); + + // Go offline then online + monitor.simulate('disconnected'); + monitor.simulate('connected'); + + expect(drainSpy).toHaveBeenCalled(); + }); }); describe('tracing', () => { @@ -877,4 +932,137 @@ describe('MetaRouterAnalyticsClient', () => { ); }); }); + + describe('maxDiskEvents validation', () => { + it('rejects negative maxDiskEvents at construction', () => { + expect( + () => + new MetaRouterAnalyticsClient({ + ...opts, + maxDiskEvents: -1, + }) + ).toThrow(/maxDiskEvents.*must be >= 0/); + }); + + it('accepts maxDiskEvents: 0 (persistence disabled)', () => { + expect( + () => + new MetaRouterAnalyticsClient({ + ...opts, + maxDiskEvents: 0, + }) + ).not.toThrow(); + }); + + it('warns when 0 < maxDiskEvents < maxQueueEvents', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 2000, + maxDiskEvents: 100, + }); + expect(warnSpy).toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining( + 'maxDiskEvents (100) is less than maxQueueEvents (2000)' + ) + ); + warnSpy.mockRestore(); + }); + + it('does not warn when maxDiskEvents === maxQueueEvents', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 500, + maxDiskEvents: 500, + }); + expect(warnSpy).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('is less than maxQueueEvents') + ); + warnSpy.mockRestore(); + }); + + it('does not warn when maxDiskEvents === 0 (disabled, not inverted)', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 2000, + maxDiskEvents: 0, + }); + expect(warnSpy).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('is less than maxQueueEvents') + ); + warnSpy.mockRestore(); + }); + + it('does not warn when maxDiskEvents > maxQueueEvents', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + new MetaRouterAnalyticsClient({ + ...opts, + maxQueueEvents: 2000, + maxDiskEvents: 10000, + }); + expect(warnSpy).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('is less than maxQueueEvents') + ); + warnSpy.mockRestore(); + }); + }); + + describe('maxDiskEvents: 0 (persistence disabled)', () => { + it('enqueue at cap drops the oldest event (ring buffer)', async () => { + const monitor = new StubNetworkMonitor('disconnected'); + const client = new MetaRouterAnalyticsClient( + { + ...opts, + maxQueueEvents: 3, + maxDiskEvents: 0, + flushIntervalSeconds: 3600, + }, + { networkMonitor: monitor } + ); + await client.init(); + (client as any).dispatcher.opts.autoFlushThreshold = 9999; + + const flushSpy = jest.spyOn( + (client as any).persistentQueue, + 'flushEventsToDisk' + ); + + for (let i = 0; i < 5; i++) { + client.track(`event_${i}`); + } + + // With 5 enqueues and cap 3, 2 oldest should have been dropped. + const queue = (client as any).queue; + expect(queue.length).toBe(3); + // No disk write — persistence is disabled. + expect(flushSpy).not.toHaveBeenCalled(); + }); + + it('background flush does not touch disk when persistence is disabled', async () => { + const client = new MetaRouterAnalyticsClient({ + ...opts, + maxDiskEvents: 0, + }); + await client.init(); + + const pq = (client as any).persistentQueue; + const flushSpy = jest.spyOn(pq, 'flushEventsToDisk'); + + (global as any).fetch = jest.fn(() => + Promise.reject(new Error('offline')) + ); + client.track('evt1'); + client.track('evt2'); + + await pq.flushToDisk(); + + expect(flushSpy).not.toHaveBeenCalled(); + }); + }); }); diff --git a/src/analytics/MetaRouterAnalyticsClient.ts b/src/analytics/MetaRouterAnalyticsClient.ts index b7aae12..7dc57d7 100644 --- a/src/analytics/MetaRouterAnalyticsClient.ts +++ b/src/analytics/MetaRouterAnalyticsClient.ts @@ -18,6 +18,7 @@ import { type NetworkReachability, type NetworkStatus, } from './utils/networkMonitor'; +import { DebouncedNetworkMonitor } from './utils/debouncedNetworkMonitor'; /** * Analytics client for MetaRouter. @@ -37,8 +38,14 @@ export class MetaRouterAnalyticsClient { private appState: AppStateStatus = AppState.currentState; private appStateSubscription: { remove?: () => void } | null = null; private identityManager: IdentityManager; - private static readonly MAX_QUEUE_SIZE = 20; - private maxQueueBytes: number = 5 * 1024 * 1024; // 5MB default + private static readonly AUTO_FLUSH_THRESHOLD = 20; + private static readonly DEFAULT_MAX_QUEUE_EVENTS = 2000; + private static readonly DEFAULT_MAX_DISK_EVENTS = 10_000; + // Byte cap is intentionally internal (parity with iOS/Android). Not a + // public option. 5MB — revisit only if a customer actually needs to tune. + private static readonly MAX_QUEUE_BYTES = 5 * 1024 * 1024; + private maxDiskEvents: number = + MetaRouterAnalyticsClient.DEFAULT_MAX_DISK_EVENTS; private dispatcher!: Dispatcher; private persistentQueue!: PersistentEventQueue; private tracingEnabled: boolean = false; @@ -82,17 +89,45 @@ export class MetaRouterAnalyticsClient { this.writeKey = writeKey; this.flushIntervalSeconds = flushIntervalSeconds ?? 10; + // Validate + normalize persistence caps. + const rawMaxDisk = + options.maxDiskEvents ?? + MetaRouterAnalyticsClient.DEFAULT_MAX_DISK_EVENTS; + if (rawMaxDisk < 0) { + throw new Error( + 'MetaRouterAnalyticsClient initialization failed: `maxDiskEvents` must be >= 0 (use 0 to disable disk persistence).' + ); + } + this.maxDiskEvents = rawMaxDisk; + + const rawMaxQueue = + options.maxQueueEvents ?? + MetaRouterAnalyticsClient.DEFAULT_MAX_QUEUE_EVENTS; + const maxQueueEvents = Math.max(1, rawMaxQueue); + + if (this.maxDiskEvents > 0 && this.maxDiskEvents < maxQueueEvents) { + warn( + `maxDiskEvents (${this.maxDiskEvents}) is less than maxQueueEvents (${maxQueueEvents}) — memory can hold more events than disk can preserve; events may be dropped during background flush` + ); + } + setDebugLogging(options.debug ?? false); this.identityManager = new IdentityManager(); - this.networkMonitor = deps?.networkMonitor ?? new NetworkMonitor(); - this.maxQueueBytes = options.maxQueueBytes ?? this.maxQueueBytes; + // Default: wrap the raw native monitor with the asymmetric debounce + // (immediate offline, 2s stable-online). If a caller injects their own + // monitor (e.g. in tests), use it as-is so they control the debounce + // behavior explicitly. + this.networkMonitor = + deps?.networkMonitor ?? new DebouncedNetworkMonitor(new NetworkMonitor()); this.dispatcher = new Dispatcher({ - maxQueueBytes: this.maxQueueBytes, - autoFlushThreshold: MetaRouterAnalyticsClient.MAX_QUEUE_SIZE, + maxEventCount: maxQueueEvents, + maxQueueBytes: MetaRouterAnalyticsClient.MAX_QUEUE_BYTES, + autoFlushThreshold: MetaRouterAnalyticsClient.AUTO_FLUSH_THRESHOLD, maxBatchSize: 100, flushIntervalSeconds: this.flushIntervalSeconds, baseRetryDelayMs: 1000, maxRetryDelayMs: 8000, + isPersistenceEnabled: () => this.maxDiskEvents > 0, isNetworkAvailable: () => this.networkStatus === 'connected', endpoint: (path) => this.endpoint(path), fetchWithTimeout: (url, init, timeoutMs) => @@ -121,6 +156,14 @@ export class MetaRouterAnalyticsClient { log, warn, error, + onCapacityOverflow: (events) => + this.persistentQueue.bufferEventsForDisk(events), + onFlushToDisk: (events) => this.persistentQueue.flushEventsToDisk(events), + onFlushComplete: () => { + if (this.networkStatus === 'connected') { + void this.persistentQueue.drainDiskToNetwork(this.dispatcher); + } + }, onScheduleFlushIn: (ms) => { (this as any).scheduleFlushIn(ms, { fromDispatcher: true }); }, @@ -130,7 +173,9 @@ export class MetaRouterAnalyticsClient { }); this.queue = this.dispatcher.getQueueRef(); - this.persistentQueue = new PersistentEventQueue(this.dispatcher); + this.persistentQueue = new PersistentEventQueue(this.dispatcher, { + maxDiskEvents: this.maxDiskEvents, + }); } /** @@ -158,7 +203,11 @@ export class MetaRouterAnalyticsClient { try { await this.identityManager.init(); - await this.persistentQueue.rehydrate(); + // Cheap existence check — memory queue starts empty. Any on-disk + // events are drained directly to the network below (if online) via + // drainDiskToNetwork, avoiding a memory spike from rehydrating a + // potentially-large backlog. + await this.persistentQueue.checkForPersistedEvents(); this.startFlushLoop(); this.setupAppStateListener(); @@ -180,21 +229,32 @@ export class MetaRouterAnalyticsClient { const wasOffline = this.networkStatus === 'disconnected'; this.networkStatus = status; + log( + `Network status changed: ${wasOffline ? 'disconnected' : 'connected'} -> ${status}` + ); if (wasOffline && status === 'connected') { - log('Network connectivity restored — resuming flush'); + log('Dispatcher resumed — device is online, triggering flush'); this.dispatcher.resetCircuitBreaker(); + // (1) Memory queue → network (existing path) void this.flush(); + // (2) Disk → network directly (no load into memory queue) + void this.persistentQueue.drainDiskToNetwork(this.dispatcher); } else if (status === 'disconnected') { - log('Network connectivity lost — pausing HTTP attempts'); + log('Dispatcher paused — device is offline'); } } ); log('MetaRouter SDK initialized'); - // Flush immediately so rehydrated events ship on cold start - // (AppState 'change' won't fire because the app is already active) - void this.flush(); + // If online at launch with on-disk events, drain them to the network. + // Memory queue starts empty, so there's no cold-start flush to run. + if ( + this.networkStatus === 'connected' && + this.persistentQueue.hasDiskData + ) { + void this.persistentQueue.drainDiskToNetwork(this.dispatcher); + } } catch (error) { this.lifecycle = 'idle'; // allow retry warn('Analytics client initialization failed:', error); @@ -271,7 +331,9 @@ export class MetaRouterAnalyticsClient { // Check if we should flush to disk based on thresholds if (this.persistentQueue.shouldFlushToDisk()) { - void this.persistentQueue.flushToDisk(); + void this.persistentQueue.flushToDisk().catch((err) => { + warn('Failed to flush queue to disk after threshold:', err); + }); } } @@ -307,8 +369,13 @@ export class MetaRouterAnalyticsClient { log('App moved to background'); this.stopFlushLoop(); this.clearNextTimer(); - await this.flush(); - await this.persistentQueue.flushToDisk(); + try { + await this.flush(); + await this.persistentQueue.flushToDisk(); + await this.persistentQueue.flushPendingDiskWrites(); + } catch (err) { + warn('Failed to persist queue while moving to background:', err); + } } if (nextState === 'active' && this.lifecycle === 'ready') { log('App moved to foreground'); @@ -505,9 +572,10 @@ export class MetaRouterAnalyticsClient { flushInFlight: d.flushInFlight, circuitState: d.circuitState, circuitRemainingMs: d.circuitRemainingMs, - maxQueueBytes: d.maxQueueBytes, + maxQueueEvents: d.maxQueueEvents, + maxDiskEvents: this.maxDiskEvents, tracingEnabled: this.tracingEnabled, - rehydratedEvents: this.persistentQueue.rehydratedEvents, + hasDiskData: this.persistentQueue.hasDiskData, networkStatus: this.networkStatus, }; } diff --git a/src/analytics/dispatcher.test.ts b/src/analytics/dispatcher.test.ts index 94bf853..d3c464e 100644 --- a/src/analytics/dispatcher.test.ts +++ b/src/analytics/dispatcher.test.ts @@ -2,12 +2,14 @@ import Dispatcher from './dispatcher'; import CircuitBreaker from './utils/circuitBreaker'; const baseOpts = () => ({ + maxEventCount: 2000, maxQueueBytes: 5 * 1024 * 1024, // 5MB autoFlushThreshold: 20, maxBatchSize: 100, flushIntervalSeconds: 3600, // keep timer quiet unless started baseRetryDelayMs: 1000, maxRetryDelayMs: 8000, + isPersistenceEnabled: () => true, isNetworkAvailable: () => true, endpoint: (p: string) => `https://example.com${p}`, fetchWithTimeout: jest.fn( @@ -48,17 +50,27 @@ describe('Dispatcher', () => { expect(fetchSpy).toHaveBeenCalled(); }); - it('respects maxQueueBytes by dropping oldest', () => { + it('respects maxQueueBytes by flushing entire queue to overflow on capacity', () => { const opts = baseOpts(); // Each event is 28 chars: {"type":"track","event":"a"} // Set cap to fit exactly 3 events (84 chars) opts.maxQueueBytes = 84; + opts.onCapacityOverflow = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); d.enqueue({ type: 'track', event: 'b' } as any); d.enqueue({ type: 'track', event: 'c' } as any); - d.enqueue({ type: 'track', event: 'd' } as any); // drops oldest - expect(d.getQueueRef().map((e: any) => e.event)).toEqual(['b', 'c', 'd']); + d.enqueue({ type: 'track', event: 'd' } as any); // triggers full flush + enqueue 'd' + // Queue should only contain the new event 'd' + expect(d.getQueueRef().map((e: any) => e.event)).toEqual(['d']); + // Overflow callback should have received 'a', 'b', 'c' + expect(opts.onCapacityOverflow).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ event: 'a' }), + expect.objectContaining({ event: 'b' }), + expect.objectContaining({ event: 'c' }), + ]) + ); }); it('batches in chunks up to maxBatchSize', async () => { @@ -157,54 +169,60 @@ describe('Dispatcher', () => { d.enqueue(makeEvent(i) as any); } - // Queue should be capped at 2000 - expect(d.getQueueRef().length).toBe(2000); - - // Verify oldest events were dropped and newest remain + // With the new flush-entire-queue overflow model, each time the cap is exceeded + // the entire queue is dropped (no onCapacityOverflow set). + // Only the most recent event(s) that fit within the cap after the last overflow remain. const queue = d.getQueueRef(); - expect((queue[0] as any).properties.id).toBe('08000'); + // Queue should not exceed cap + expect(d.getQueueSizeBytes()).toBeLessThanOrEqual(eventSize * 2000); + // The last event should always be present expect((queue[queue.length - 1] as any).properties.id).toBe('09999'); - // Verify all IDs in queue are contiguous - for (let i = 0; i < 2000; i++) { - expect((queue[i] as any).properties.id).toBe( - String(8000 + i).padStart(5, '0') - ); - } - - // Verify warn was called 8000 times (once per dropped event) - expect((opts.warn as jest.Mock).mock.calls.length).toBe(8000); - expect((opts.warn as jest.Mock).mock.calls[0][0]).toContain( - 'Queue cap reached' - ); + // Verify warn was called for each overflow (entire queue dropped each time) + expect( + (opts.warn as jest.Mock).mock.calls.some((c) => + String(c[0]).includes('Queue cap reached') + ) + ).toBe(true); }); it('gradually recovers maxBatchSize after 413-induced reduction on subsequent successes', async () => { - const opts = baseOpts(); - opts.maxBatchSize = 100; - opts.autoFlushThreshold = 9999; - let callCount = 0; - opts.fetchWithTimeout = jest.fn(async (_url?: string, _init?: any) => { - callCount++; - // First call: 413 to trigger halving - if (callCount === 1) - return { ok: false, status: 413, headers: { get: () => null } } as any; - return { ok: true, status: 200 } as any; - }); + jest.useFakeTimers(); + try { + const opts = baseOpts(); + opts.maxBatchSize = 100; + opts.autoFlushThreshold = 9999; + let callCount = 0; + opts.fetchWithTimeout = jest.fn(async (_url?: string, _init?: any) => { + callCount++; + // First call: 413 to trigger halving + if (callCount === 1) + return { + ok: false, + status: 413, + headers: { get: () => null }, + } as any; + return { ok: true, status: 200 } as any; + }); - const d = new Dispatcher(opts); + const d = new Dispatcher(opts); - // Enqueue enough events and flush to trigger the 413 - for (let i = 0; i < 5; i++) - d.enqueue({ type: 'track', event: `e${i}` } as any); - await d.flush(); + // Enqueue enough events and flush to trigger the 413 + for (let i = 0; i < 5; i++) + d.enqueue({ type: 'track', event: `e${i}` } as any); + await d.flush(); - // After 413, maxBatchSize should have been halved to 50 - expect(d.getDebugInfo().maxBatchSize).toBe(50); + // After 413, maxBatchSize should have been halved to 50 + expect(d.getDebugInfo().maxBatchSize).toBe(50); - // Flush again — events succeed, so batch size should recover: min(50*2, 100) = 100 - await d.flush(); - expect(d.getDebugInfo().maxBatchSize).toBe(100); + // Advance past the scheduled retry (500ms) — the internal retry fires + // success, which recovers batch size: min(50*2, 100) = 100. + await jest.advanceTimersByTimeAsync(500); + + expect(d.getDebugInfo().maxBatchSize).toBe(100); + } finally { + jest.useRealTimers(); + } }); it('does not recover maxBatchSize beyond initialMaxBatchSize', async () => { @@ -248,11 +266,12 @@ describe('Dispatcher', () => { ]); }); - it('enqueueFront respects maxQueueBytes by dropping oldest from front-loaded batch', () => { + it('enqueueFront flushes entire queue to overflow when exceeding capacity', () => { const opts = baseOpts(); // Each event is 28 chars — set cap to fit exactly 3 (84 chars) opts.maxQueueBytes = 84; opts.autoFlushThreshold = 9999; + opts.onCapacityOverflow = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'a' } as any); d.enqueue({ type: 'track', event: 'b' } as any); @@ -263,10 +282,14 @@ describe('Dispatcher', () => { { type: 'track', event: 'z' } as any, ]); - // Total would be 5×28=140 chars, cap is 84. After prepend: [x, y, z, a, b] -> drop oldest 2 -> [z, a, b] - const queue = d.getQueueRef(); - expect(queue.length).toBe(3); - expect(queue.map((e: any) => e.event)).toEqual(['z', 'a', 'b']); + // Total would be 5×28=140 chars, cap is 84. Entire merged queue flushed to overflow. + expect(opts.onCapacityOverflow).toHaveBeenCalledTimes(1); + const flushed = (opts.onCapacityOverflow as jest.Mock).mock.calls[0][0]; + expect(flushed.length).toBe(5); + expect(flushed.map((e: any) => e.event)).toEqual(['x', 'y', 'z', 'a', 'b']); + + // Queue should be empty after overflow flush + expect(d.getQueueRef().length).toBe(0); }); it('getQueueSizeBytes returns approximate serialized size', () => { @@ -385,6 +408,9 @@ describe('Dispatcher', () => { it('circuit breaker does NOT reset while still connected but failing', async () => { const opts = baseOpts(); + // Drop chunks on failure (don't schedule retry) so each manual flush + // can run and the circuit sees all three failures. + opts.isOperational = () => false; opts.fetchWithTimeout = jest.fn( async () => ({ ok: false, status: 500, headers: { get: () => null } }) as any @@ -408,9 +434,10 @@ describe('Dispatcher', () => { expect(d.getDebugInfo().isNetworkAvailable).toBe(false); }); - it('offline log warns with queue count', async () => { + it('offline log warns with flushed event count when onFlushToDisk is set', async () => { const opts = baseOpts(); opts.isNetworkAvailable = () => false; + opts.onFlushToDisk = jest.fn(); const d = new Dispatcher(opts); d.enqueue({ type: 'track', event: 'e1' } as any); @@ -421,10 +448,355 @@ describe('Dispatcher', () => { (opts.warn as jest.Mock).mock.calls.some( (c) => String(c[0]).includes('Offline') && + String(c[0]).includes('flushed') && String(c[0]).includes('2 event(s)') ) ).toBe(true); }); + + it('restores events to memory if offline disk flush fails', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.onFlushToDisk = jest.fn(async () => { + throw new Error('disk full'); + }); + opts.autoFlushThreshold = 9999; + const d = new Dispatcher(opts); + + d.enqueue({ type: 'track', event: 'e1' } as any); + d.enqueue({ type: 'track', event: 'e2' } as any); + await d.flush(); + + expect(d.getQueueRef().map((e: any) => e.event)).toEqual(['e1', 'e2']); + expect( + (opts.warn as jest.Mock).mock.calls.some((c) => + String(c[0]).includes('restored to memory') + ) + ).toBe(true); + }); + + it('offline log warns with queue count when no onFlushToDisk', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + const d = new Dispatcher(opts); + + d.enqueue({ type: 'track', event: 'e1' } as any); + d.enqueue({ type: 'track', event: 'e2' } as any); + await d.flush(); + + expect( + (opts.warn as jest.Mock).mock.calls.some( + (c) => + String(c[0]).includes('Offline') && + String(c[0]).includes('2 event(s)') + ) + ).toBe(true); + }); + }); + + describe('onCapacityOverflow callback', () => { + it('fires with entire queue when capacity is hit', () => { + const opts = baseOpts(); + const overflowEvents: any[][] = []; + opts.onCapacityOverflow = jest.fn((events: any[]) => { + overflowEvents.push(events); + }); + opts.maxQueueBytes = 84; // fits ~3 small events + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + d.enqueue({ type: 'track', event: 'c' } as any); + d.enqueue({ type: 'track', event: 'd' } as any); // flushes a,b,c to overflow + + expect(opts.onCapacityOverflow).toHaveBeenCalledTimes(1); + expect(overflowEvents[0].length).toBe(3); + expect(overflowEvents[0].map((e: any) => e.event)).toEqual([ + 'a', + 'b', + 'c', + ]); + + // Queue should only contain the new event + expect(d.getQueueRef().length).toBe(1); + expect((d.getQueueRef()[0] as any).event).toBe('d'); + }); + + it('does not fire when queue is within capacity', () => { + const opts = baseOpts(); + opts.onCapacityOverflow = jest.fn(); + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + expect(opts.onCapacityOverflow).not.toHaveBeenCalled(); + }); + + it('fires when the event-count cap is hit (even when well under byte cap)', () => { + const opts = baseOpts(); + const overflowEvents: any[][] = []; + opts.onCapacityOverflow = jest.fn((events: any[]) => { + overflowEvents.push(events); + }); + opts.maxEventCount = 3; + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + for (let i = 0; i < 3; i++) { + d.enqueue({ type: 'track', event: `e${i}` } as any); + } + // 4th event would exceed count cap — the existing 3 are spliced out. + d.enqueue({ type: 'track', event: 'e3' } as any); + + expect(opts.onCapacityOverflow).toHaveBeenCalledTimes(1); + expect(overflowEvents[0].length).toBe(3); + expect(d.getQueueRef().length).toBe(1); + expect((d.getQueueRef()[0] as any).event).toBe('e3'); + }); + + it('drops oldest (ring buffer) when persistence is disabled', () => { + const opts = baseOpts(); + opts.onCapacityOverflow = jest.fn(); + opts.isPersistenceEnabled = () => false; + opts.maxEventCount = 3; + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + for (let i = 0; i < 5; i++) { + d.enqueue({ type: 'track', event: `e${i}` } as any); + } + + // Queue still size 3 (ring buffer); the 2 oldest were dropped one by one. + expect(d.getQueueRef().length).toBe(3); + const events = d.getQueueRef().map((e: any) => e.event); + expect(events).toEqual(['e2', 'e3', 'e4']); + // No disk splice happened. + expect(opts.onCapacityOverflow).not.toHaveBeenCalled(); + }); + + it('resets queue bytes to 0 after overflow', () => { + const opts = baseOpts(); + opts.onCapacityOverflow = jest.fn(); + const eventSize = new TextEncoder().encode( + JSON.stringify({ type: 'track', event: 'x' }) + ).byteLength; + opts.maxQueueBytes = eventSize; // fits exactly 1 event + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); // overflows 'a', enqueues 'b' + + // Queue should have just 'b', size should be 1 event's worth + expect(d.getQueueRef().length).toBe(1); + expect(d.getQueueSizeBytes()).toBe(eventSize); + }); + }); + + describe('onFlushToDisk callback', () => { + it('fires when flush is called while offline', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.onFlushToDisk = jest.fn(); + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + await d.flush(); + + expect(opts.onFlushToDisk).toHaveBeenCalledTimes(1); + expect(opts.onFlushToDisk).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ event: 'a' }), + expect.objectContaining({ event: 'b' }), + ]) + ); + // Queue should be empty after flushing to disk + expect(d.getQueueRef().length).toBe(0); + }); + + it('does not fire when online', async () => { + const opts = baseOpts(); + opts.onFlushToDisk = jest.fn(); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + await d.flush(); + + expect(opts.onFlushToDisk).not.toHaveBeenCalled(); + }); + + it('awaits onFlushToDisk before resolving offline flush', async () => { + const opts = baseOpts(); + opts.isNetworkAvailable = () => false; + opts.autoFlushThreshold = 9999; + let persisted = false; + opts.onFlushToDisk = jest.fn(async () => { + await Promise.resolve(); + persisted = true; + }); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + + await d.flush(); + + expect(persisted).toBe(true); + expect(d.getQueueRef().length).toBe(0); + }); + }); + + describe('onFlushComplete callback', () => { + it('fires after successful online flush that empties the queue', async () => { + const opts = baseOpts(); + opts.onFlushComplete = jest.fn(); + opts.autoFlushThreshold = 9999; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + + await d.flush(); + + expect(opts.onFlushComplete).toHaveBeenCalledTimes(1); + }); + + it('does not fire on failed flush', async () => { + const opts = baseOpts(); + opts.onFlushComplete = jest.fn(); + opts.autoFlushThreshold = 9999; + opts.fetchWithTimeout = jest.fn(async () => ({ + ok: false, + status: 500, + headers: { get: () => null }, + })) as any; + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + + await d.flush(); + + expect(opts.onFlushComplete).not.toHaveBeenCalled(); + }); + + it('does not fire when all batches are dropped as 4xx (no 2xx observed)', async () => { + const opts = baseOpts(); + opts.onFlushComplete = jest.fn(); + opts.autoFlushThreshold = 9999; + opts.fetchWithTimeout = jest.fn( + async () => + ({ ok: false, status: 400, headers: { get: () => null } }) as any + ); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + d.enqueue({ type: 'track', event: 'b' } as any); + + await d.flush(); + + expect(opts.onFlushComplete).not.toHaveBeenCalled(); + expect(d.getQueueRef().length).toBe(0); + }); + }); + + describe('retry-backoff churn guard', () => { + it('short-circuits flush while a retry timer is armed', async () => { + const opts = baseOpts(); + opts.autoFlushThreshold = 9999; + opts.fetchWithTimeout = jest.fn( + async () => + ({ ok: false, status: 500, headers: { get: () => null } }) as any + ); + + const d = new Dispatcher(opts); + d.enqueue({ type: 'track', event: 'a' } as any); + await d.flush(); + + const fetchSpy = opts.fetchWithTimeout as jest.Mock; + const callsAfterRetryArmed = fetchSpy.mock.calls.length; + expect(opts.onScheduleFlushIn).toHaveBeenCalled(); + + // External callers (periodic timer, enqueue auto-flush, onFlushComplete) + // must not bypass the backoff window. + await d.flush(); + await d.flush(); + await d.flush(); + + expect(fetchSpy.mock.calls.length).toBe(callsAfterRetryArmed); + }); + }); + + describe('sendBatchDirect', () => { + it('returns statusCode on successful send', async () => { + const opts = baseOpts(); + const d = new Dispatcher(opts); + + const result = await d.sendBatchDirect([ + { type: 'track', event: 'e1' } as any, + ]); + expect(result).toEqual({ statusCode: 200 }); + expect(opts.fetchWithTimeout).toHaveBeenCalled(); + }); + + it('returns statusCode on HTTP error', async () => { + const opts = baseOpts(); + opts.fetchWithTimeout = jest.fn(async () => ({ + ok: false, + status: 500, + })) as any; + const d = new Dispatcher(opts); + + const result = await d.sendBatchDirect([ + { type: 'track', event: 'e1' } as any, + ]); + expect(result).toEqual({ statusCode: 500 }); + }); + + it('returns null on network error', async () => { + const opts = baseOpts(); + opts.fetchWithTimeout = jest.fn(async () => { + throw new Error('Network unavailable'); + }); + const d = new Dispatcher(opts); + + const result = await d.sendBatchDirect([ + { type: 'track', event: 'e1' } as any, + ]); + expect(result).toBeNull(); + }); + + it('does not affect the memory queue', async () => { + const opts = baseOpts(); + opts.autoFlushThreshold = 9999; + const d = new Dispatcher(opts); + + d.enqueue({ type: 'track', event: 'queued' } as any); + await d.sendBatchDirect([{ type: 'track', event: 'direct' } as any]); + + // Memory queue unchanged + expect(d.getQueueRef().length).toBe(1); + expect((d.getQueueRef()[0] as any).event).toBe('queued'); + }); + + it('stamps sentAt on batch events', async () => { + const opts = baseOpts(); + let sentBody: any = null; + opts.fetchWithTimeout = jest.fn(async (_url?: string, init?: any) => { + sentBody = JSON.parse(init.body); + return { ok: true, status: 200 } as any; + }); + const d = new Dispatcher(opts); + + await d.sendBatchDirect([{ type: 'track', event: 'e1' } as any]); + + expect(sentBody.batch[0].sentAt).toBeDefined(); + }); }); it('stress test: all ~2K queued events successfully transmit in batches', async () => { diff --git a/src/analytics/dispatcher.ts b/src/analytics/dispatcher.ts index ee4ee67..8e5576e 100644 --- a/src/analytics/dispatcher.ts +++ b/src/analytics/dispatcher.ts @@ -3,14 +3,25 @@ import type CircuitBreaker from './utils/circuitBreaker'; export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN'; +export interface NetworkResponse { + statusCode: number; +} + export interface DispatcherOptions { - maxQueueBytes: number; // byte-based cap on memory queue (~5MB default) + maxEventCount: number; // event-count cap on memory queue (default 2000) + maxQueueBytes: number; // byte-based cap on memory queue (~5MB default, internal) autoFlushThreshold: number; // e.g., 20 maxBatchSize: number; // initial max; may shrink on 413 flushIntervalSeconds: number; baseRetryDelayMs: number; // retry floor base delay (default 1000) maxRetryDelayMs: number; // retry floor cap (default 8000) + /** + * Whether disk persistence is enabled. When false, capacity overflow drops + * the oldest event (ring buffer) instead of splicing the whole queue to + * the onCapacityOverflow handler, since there is no disk to spill to. + */ + isPersistenceEnabled: () => boolean; isNetworkAvailable: () => boolean; // returns false when offline endpoint: (path: string) => string; @@ -28,6 +39,9 @@ export interface DispatcherOptions { warn: (...args: any[]) => void; error: (...args: any[]) => void; + onCapacityOverflow?: (events: EventPayload[]) => void | Promise; // called with entire queue when cap is hit + onFlushToDisk?: (events: EventPayload[]) => void | Promise; // called when flush triggers while offline — persists queue to disk + onFlushComplete?: () => void; // fires after successful online flush to trigger disk drain onScheduleFlushIn?: (ms: number) => void; // optional notification for tests/metrics onFatalConfig?: () => void; // 401/403/404 handler } @@ -56,6 +70,25 @@ export default class Dispatcher { return this.queue; } + /** + * Drain the in-memory queue entirely. Returns all events and resets the + * byte counter. Used by PersistentEventQueue to persist memory to disk + * without leaving duplicates behind. + */ + drainQueue(): EventPayload[] { + const events = this.queue.splice(0); + this.queueSizeBytes = 0; + return events; + } + + /** + * Restore events to the front without applying capacity overflow policy. + * Used only after a failed durability handoff so events are not lost. + */ + restoreQueueFront(events: EventPayload[]): void { + this.requeueChunk(events); + } + start(): void { if (this.flushTimer) clearInterval(this.flushTimer); this.flushTimer = setInterval( @@ -126,17 +159,73 @@ export default class Dispatcher { return Dispatcher.encoder.encode(JSON.stringify(event)).byteLength; } + private handleCapacityOverflow(triggerDesc: string): void { + // Persistence disabled (maxDiskEvents === 0): ring-buffer — drop oldest + // event one at a time instead of draining the whole queue, since there + // is no disk to spill to. + if (!this.opts.isPersistenceEnabled()) { + const dropped = this.queue.shift(); + if (dropped !== undefined) { + this.queueSizeBytes -= this.estimateEventSize(dropped); + if (this.queueSizeBytes < 0) this.queueSizeBytes = 0; + this.opts.warn( + `Queue cap reached (${triggerDesc}) — dropped oldest event` + ); + } + return; + } + + // Persistence enabled: splice the whole queue to the disk handler. + if (this.opts.onCapacityOverflow) { + const flushed = this.queue.splice(0); + this.queueSizeBytes = 0; + try { + const result = this.opts.onCapacityOverflow(flushed); + void Promise.resolve(result).catch((err) => { + this.requeueChunk(flushed); + this.opts.warn( + 'Capacity overflow persistence failed — restored events to memory', + err + ); + }); + } catch (err) { + this.requeueChunk(flushed); + this.opts.warn( + 'Capacity overflow persistence failed — restored events to memory', + err + ); + } + } else { + this.opts.warn( + `Queue cap reached — dropping ${this.queue.length} event(s) (no overflow handler)` + ); + this.queue.length = 0; + this.queueSizeBytes = 0; + } + } + + private isOverCap(extraBytes: number = 0, extraCount: number = 0): boolean { + return ( + this.queue.length + extraCount > this.opts.maxEventCount || + this.queueSizeBytes + extraBytes > this.opts.maxQueueBytes + ); + } + enqueue(event: EventPayload): void { const eventSize = this.estimateEventSize(event); - // Hard cap: drop oldest until there's room (byte limit) - while ( - this.queue.length > 0 && - this.queueSizeBytes + eventSize > this.opts.maxQueueBytes - ) { - const dropped = this.queue.shift(); - if (dropped) this.queueSizeBytes -= this.estimateEventSize(dropped); - this.opts.warn('Queue cap reached — dropped oldest event'); + // Enforce cap: if adding this event would exceed count OR byte limit, + // flush/evict first so the new event still fits. + while (this.queue.length > 0 && this.isOverCap(eventSize, 1)) { + this.handleCapacityOverflow( + this.queueSizeBytes + eventSize > this.opts.maxQueueBytes + ? 'bytes' + : 'count' + ); + if (!this.opts.isPersistenceEnabled() && this.queue.length === 0) break; + // When persistence IS enabled, handleCapacityOverflow drains the whole + // queue so the loop exits naturally. + if (this.opts.isPersistenceEnabled()) break; } this.opts.log( @@ -162,14 +251,14 @@ export default class Dispatcher { this.queueSizeBytes += this.estimateEventSize(e); } - // Enforce cap: drop oldest (from front) until within byte limit - while ( - this.queue.length > 0 && - this.queueSizeBytes > this.opts.maxQueueBytes - ) { - const dropped = this.queue.shift(); - if (dropped) this.queueSizeBytes -= this.estimateEventSize(dropped); - this.opts.warn('Queue cap reached — dropped oldest event'); + // Enforce cap: drain to disk (or ring-buffer drop-oldest in 0-mode). + while (this.isOverCap()) { + const before = this.queue.length; + this.handleCapacityOverflow( + this.queueSizeBytes > this.opts.maxQueueBytes ? 'bytes' : 'count' + ); + if (this.queue.length === before) break; // safety: avoid infinite loop + if (this.opts.isPersistenceEnabled()) break; // drained entire queue } if (this.queue.length >= this.opts.autoFlushThreshold) { @@ -215,14 +304,34 @@ export default class Dispatcher { async flush(): Promise { if (!this.queue.length) return; if (this.flushInFlight) return this.flushInFlight; + // A retry is armed — let it fire on its own schedule instead of + // launching immediately and bypassing the backoff window. The scheduled + // callback nils nextTimer before calling flush(), so it passes through. + if (this.nextTimer) return; if (!this.opts.canSend()) return; const doFlush = async () => { while (this.queue.length) { if (!this.opts.isNetworkAvailable()) { - this.opts.warn( - `Offline — pausing HTTP attempts, ${this.queue.length} event(s) queued` - ); + if (this.opts.onFlushToDisk && this.opts.isPersistenceEnabled()) { + const flushed = this.drainQueue(); + try { + await this.opts.onFlushToDisk(flushed); + this.opts.warn( + `Offline — flushed ${flushed.length} event(s) to disk` + ); + } catch (err) { + this.requeueChunk(flushed); + this.opts.warn( + `Offline — failed to flush ${flushed.length} event(s) to disk; restored to memory`, + err + ); + } + } else { + this.opts.warn( + `Offline — pausing HTTP attempts, ${this.queue.length} event(s) queued` + ); + } return; } @@ -312,11 +421,7 @@ export default class Dispatcher { } if (s === 401 || s === 403 || s === 404) { - this.opts.error(`Fatal config error ${s}. Disabling client.`); - this.queue.length = 0; - this.queueSizeBytes = 0; - this.stop(); - this.opts.onFatalConfig?.(); + this.handleFatalConfig(s); return; } @@ -359,6 +464,11 @@ export default class Dispatcher { ); } this.opts.log('API call successful'); + + // If queue is now empty after this successful batch, fire onFlushComplete + if (this.queue.length === 0 && this.opts.isNetworkAvailable()) { + this.opts.onFlushComplete?.(); + } } catch (err) { this.circuit.onFailure(); this.consecutiveRetries += 1; @@ -394,6 +504,61 @@ export default class Dispatcher { this.consecutiveRetries = 0; } + /** + * Coordinate shutdown on a fatal config error (401/403/404). + * Called by the main flush path and by the disk drain so both paths + * converge to the same "disable client" signal. + */ + handleFatalConfig(statusCode: number): void { + this.opts.error(`Fatal config error ${statusCode}. Disabling client.`); + this.queue.length = 0; + this.queueSizeBytes = 0; + this.stop(); + this.opts.onFatalConfig?.(); + } + + /** + * Send a batch directly to network, bypassing the memory queue. + * Used by the disk drain to send events without loading them into the queue. + * Returns { statusCode } on HTTP response, null on network/transport error. + */ + async sendBatchDirect( + events: EventPayload[] + ): Promise { + try { + const nowIso = new Date().toISOString(); + const batch = events.map((e) => ({ ...(e as any), sentAt: nowIso })); + + const headers: Record = { + 'Content-Type': 'application/json', + }; + if (this.opts.isTracingEnabled()) { + headers.Trace = 'true'; + } + + const res = await this.opts.fetchWithTimeout( + this.opts.endpoint('/v1/batch'), + { + method: 'POST', + headers, + body: JSON.stringify({ batch }), + }, + 8000 + ); + + if (res.ok) { + this.opts.log(`Direct batch send successful (${events.length} events)`); + } else { + this.opts.warn(`Direct batch send failed with status ${res.status}`); + } + + return { statusCode: res.status }; + } catch (err) { + this.opts.warn(`Direct batch send failed: ${(err as any)?.message}`); + return null; + } + } + getDebugInfo() { return { queueLength: this.queue.length, @@ -402,7 +567,7 @@ export default class Dispatcher { flushInFlight: !!this.flushInFlight, circuitState: this.circuit.getState(), circuitRemainingMs: this.circuit.remainingCooldownMs(), - maxQueueBytes: this.opts.maxQueueBytes, + maxQueueEvents: this.opts.maxEventCount, maxBatchSize: this.maxBatchSize, consecutiveRetries: this.consecutiveRetries, isNetworkAvailable: this.opts.isNetworkAvailable(), diff --git a/src/analytics/init.test.ts b/src/analytics/init.test.ts index 3ecdc90..a256a2f 100644 --- a/src/analytics/init.test.ts +++ b/src/analytics/init.test.ts @@ -148,7 +148,7 @@ describe('createAnalyticsClient', () => { }); }); - it('supports reconfiguration after reset with new maxQueueBytes', async () => { + it('supports reconfiguration after reset with new maxQueueEvents', async () => { // Note: This test documents the proper pattern: // 1. await reset() before reconfiguring // 2. The warning added in createAnalyticsClient catches forgotten awaits @@ -168,42 +168,42 @@ describe('createAnalyticsClient', () => { const { createAnalyticsClient } = require('./init'); - // First client with maxQueueBytes: 3MB + // First client with maxQueueEvents: 1000 const client1 = createAnalyticsClient({ ...opts, - maxQueueBytes: 3 * 1024 * 1024, + maxQueueEvents: 1000, }); // Wait for init to complete await new Promise((resolve) => setTimeout(resolve, 100)); let debug1 = await client1.getDebugInfo(); - expect(debug1?.maxQueueBytes).toBe(3 * 1024 * 1024); + expect(debug1?.maxQueueEvents).toBe(1000); // Properly await reset before reconfiguring await client1.reset(); - // Create new client with maxQueueBytes: 5MB + // Create new client with maxQueueEvents: 3000 const client2 = createAnalyticsClient({ ...opts, - maxQueueBytes: 5 * 1024 * 1024, + maxQueueEvents: 3000, }); // Wait for new init to complete await new Promise((resolve) => setTimeout(resolve, 100)); const debug2 = await client2.getDebugInfo(); - expect(debug2?.maxQueueBytes).toBe(5 * 1024 * 1024); + expect(debug2?.maxQueueEvents).toBe(3000); }); it('warns if reconfiguration attempted without awaiting reset', async () => { const { createAnalyticsClient } = require('./init'); const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); - // First client with maxQueueBytes: 3MB + // First client with maxQueueEvents: 1000 const client1 = createAnalyticsClient({ ...opts, - maxQueueBytes: 3 * 1024 * 1024, + maxQueueEvents: 1000, }); await new Promise((resolve) => setTimeout(resolve, 50)); @@ -214,7 +214,7 @@ describe('createAnalyticsClient', () => { // Immediately try to create with new config const client2 = createAnalyticsClient({ ...opts, - maxQueueBytes: 5 * 1024 * 1024, + maxQueueEvents: 3000, }); // Should warn about config change diff --git a/src/analytics/persistence/NativeQueueStorage.ts b/src/analytics/persistence/NativeQueueStorage.ts index 3d73a1a..3772d17 100644 --- a/src/analytics/persistence/NativeQueueStorage.ts +++ b/src/analytics/persistence/NativeQueueStorage.ts @@ -2,14 +2,19 @@ import { NativeModules } from 'react-native'; import { warn } from '../utils/logger'; /** - * Native bridge for queue snapshot persistence. + * Native bridge for the single queue disk file (queue.v1.json). * - * Expected native module contract: - * - readSnapshot(): Promise — returns file contents or null - * - writeSnapshot(data: string): Promise — overwrites file with data - * - deleteSnapshot(): Promise — deletes file if it exists + * Native module contract (iOS + Android): + * - readSnapshot(): Promise — full contents, or null if no file + * - writeSnapshot(data: string): Promise — atomic overwrite + * - deleteSnapshot(): Promise — delete if present + * - exists(): Promise — cheap existence check + * + * Append / merge / cap logic lives in JS (PersistentEventQueue) rather than + * native so the policy stays in one place and is easy to test. */ interface NativeQueueStorageModule { + exists(): Promise; readSnapshot(): Promise; writeSnapshot(data: string): Promise; deleteSnapshot(): Promise; @@ -28,6 +33,17 @@ function getModule(): NativeQueueStorageModule | null { return mod; } +export async function exists(): Promise { + const mod = getModule(); + if (!mod?.exists) return false; + try { + return await mod.exists(); + } catch (err) { + warn('Failed to check queue snapshot existence:', err); + return false; + } +} + export async function readSnapshot(): Promise { const mod = getModule(); if (!mod) return null; @@ -35,17 +51,22 @@ export async function readSnapshot(): Promise { return await mod.readSnapshot(); } catch (err) { warn('Failed to read queue snapshot from disk:', err); - return null; + throw err; } } export async function writeSnapshot(data: string): Promise { const mod = getModule(); - if (!mod) return; + if (!mod) { + throw new Error( + 'MetaRouterQueueStorage native module is not available; cannot write queue snapshot.' + ); + } try { await mod.writeSnapshot(data); } catch (err) { warn('Failed to write queue snapshot to disk:', err); + throw err; } } @@ -56,5 +77,6 @@ export async function deleteSnapshot(): Promise { await mod.deleteSnapshot(); } catch (err) { warn('Failed to delete queue snapshot from disk:', err); + throw err; } } diff --git a/src/analytics/persistence/PersistentEventQueue.ts b/src/analytics/persistence/PersistentEventQueue.ts index b8e07e4..6b5aebb 100644 --- a/src/analytics/persistence/PersistentEventQueue.ts +++ b/src/analytics/persistence/PersistentEventQueue.ts @@ -1,5 +1,7 @@ +import type { EventPayload } from '../types'; import type Dispatcher from '../dispatcher'; import { + exists as nativeExists, readSnapshot, writeSnapshot, deleteSnapshot as nativeDeleteSnapshot, @@ -9,136 +11,210 @@ import { FLUSH_THRESHOLD_BYTES, FLUSH_THRESHOLD_COUNT, DEFAULT_EVENT_TTL_MS, + DEFAULT_MAX_DISK_EVENTS, type QueueSnapshot, } from './types'; +import { + ResponseCategory, + categorizeResponse, +} from '../utils/responseCategory'; import { log, warn } from '../utils/logger'; /** - * Module-level guard: rehydrate at most once per process lifetime. - * Prevents duplicate rehydration if init() is called multiple times - * (e.g. after a reset + reinit in the same session). + * Default batch size for draining disk events to network. + * May be halved on 413 responses. */ -let hasRehydrated = false; +const DRAIN_BATCH_SIZE = 100; +/** + * Memory + disk coordination for the event queue. + * + * Single disk file (queue.v1.json) backs both crash-safety (background flush) + * and offline overflow. All native file mutations run through one serialized + * lane so append, drain, and delete operations cannot interleave. + * + * On boot, {@link checkForPersistedEvents} does a cheap existence check and + * sets {@link hasDiskData}. The in-memory queue starts empty; when online, + * the dispatcher drains disk events directly to the network via + * {@link drainDiskToNetwork} instead of rehydrating into memory. + */ export class PersistentEventQueue { private readonly dispatcher: Dispatcher; - private flushInFlight: Promise | null = null; - private _rehydratedEvents: number = 0; + private readonly maxDiskEvents: number; + private _hasDiskData: boolean = false; - constructor(dispatcher: Dispatcher) { + /** Serializes every native file mutation: append, drain rewrite/delete, reset delete. */ + private diskMutation: Promise = Promise.resolve(); + /** + * Capacity overflow happens from a synchronous enqueue path. These events + * are owned here until a native write succeeds, so a failed write does not + * silently discard them from JS memory. + */ + private pendingDiskEvents: EventPayload[] = []; + private pendingFlushInFlight: Promise | null = null; + /** Guards drainDiskToNetwork (read-then-delete semantics). */ + private drainInFlight: Promise | null = null; + + constructor(dispatcher: Dispatcher, opts?: { maxDiskEvents?: number }) { this.dispatcher = dispatcher; + this.maxDiskEvents = opts?.maxDiskEvents ?? DEFAULT_MAX_DISK_EVENTS; } - get rehydratedEvents(): number { - return this._rehydratedEvents; + private enqueueDiskMutation(fn: () => Promise): Promise { + const run = this.diskMutation.then(fn, fn); + this.diskMutation = run.then( + () => undefined, + () => undefined + ); + return run; + } + + private waitForDiskIdle(): Promise { + return this.diskMutation; + } + + /** True when a disk snapshot existed at boot or was written since. */ + get hasDiskData(): boolean { + return this._hasDiskData; } /** - * Rehydrate events from disk into the in-memory queue. - * Only runs once per process lifetime. + * Boot-time cheap existence check. + * Sets {@link hasDiskData} without reading or parsing the snapshot file, + * so we can decide whether to trigger a drain without paying the cost of + * loading a potentially-large snapshot into memory. */ - async rehydrate(): Promise { - if (hasRehydrated) { - log('Rehydration already completed this process — skipping'); - return; - } - hasRehydrated = true; - + async checkForPersistedEvents(): Promise { try { - const raw = await readSnapshot(); - if (!raw) { - log('No queue snapshot found on disk'); - return; + this._hasDiskData = await nativeExists(); + if (this._hasDiskData) { + log('Persisted events detected on disk — drain pending'); } + return this._hasDiskData; + } catch (err) { + warn('Failed to check for persisted events:', err); + this._hasDiskData = false; + return false; + } + } - let snapshot: QueueSnapshot; - try { - snapshot = JSON.parse(raw); - } catch { - warn('Queue snapshot is corrupt JSON — discarding'); - await nativeDeleteSnapshot(); - return; - } + /** + * Flush the in-memory queue to disk (append + cap). + * Called on app background and when the flush-to-disk threshold is hit. + * Serialized against other disk writes. + * + * No-op when persistence is disabled (maxDiskEvents === 0). Memory queue + * is left intact so a subsequent foreground flush can still try to send + * those events; on app kill they are lost (documented tradeoff). + */ + async flushToDisk(): Promise { + if (this.maxDiskEvents === 0) return; + const queue = this.dispatcher.getQueueRef(); + if (queue.length === 0) return; + const events = this.dispatcher.drainQueue(); + try { + await this.flushEventsToDisk(events); + } catch (err) { + this.dispatcher.restoreQueueFront(events); + warn('Failed to flush queue to disk — restored events to memory:', err); + throw err; + } + } - if (snapshot.version !== SNAPSHOT_VERSION) { - warn( - `Queue snapshot version ${snapshot.version} is not supported (expected ${SNAPSHOT_VERSION}) — discarding` - ); - await nativeDeleteSnapshot(); - return; - } + /** + * Append explicit events to the disk store (read-merge-cap-write). + * Called by lifecycle/offline paths that can await disk durability. + * Serialized against every other disk mutation. + * + * No-op when persistence is disabled (maxDiskEvents === 0). + */ + async flushEventsToDisk(events: EventPayload[]): Promise { + if (events.length === 0) return; + if (this.maxDiskEvents === 0) return; - if (!Array.isArray(snapshot.events) || snapshot.events.length === 0) { - log('Queue snapshot has no events — discarding'); - await nativeDeleteSnapshot(); - return; - } + await this.flushPendingDiskWrites(); + return this.enqueueDiskMutation(() => this._doFlushEventsToDisk(events)); + } - // Filter out events older than TTL - const now = Date.now(); - const cutoff = now - DEFAULT_EVENT_TTL_MS; - const fresh = snapshot.events.filter((e) => { - const ts = (e as any).timestamp; - if (!ts) return true; // keep events without timestamp - const eventTime = new Date(ts).getTime(); - return !isNaN(eventTime) && eventTime > cutoff; - }); - - const expired = snapshot.events.length - fresh.length; - if (expired > 0) { - warn(`Dropped ${expired} expired events (older than 7 days)`); - } + /** + * Own events from synchronous overflow paths until they are durable. + * The write is started immediately but errors are retained in the pending + * buffer for a later retry instead of being dropped. + */ + bufferEventsForDisk(events: EventPayload[]): void { + if (events.length === 0 || this.maxDiskEvents === 0) return; + this.appendPendingDiskEvents(events); + void this.flushPendingDiskWrites().catch((err) => { + warn('Failed to flush pending disk events:', err); + }); + } - if (fresh.length === 0) { - log('All snapshot events expired — discarding'); - await nativeDeleteSnapshot(); - return; - } + async flushPendingDiskWrites(): Promise { + if (this.maxDiskEvents === 0) { + this.pendingDiskEvents = []; + return; + } + if (this.pendingFlushInFlight) return this.pendingFlushInFlight; - this.dispatcher.enqueueFront(fresh); - this._rehydratedEvents = fresh.length; - log(`Rehydrated ${fresh.length} events from disk`); + this.pendingFlushInFlight = this._flushPendingDiskWrites().finally(() => { + this.pendingFlushInFlight = null; + }); + return this.pendingFlushInFlight; + } - // Clean up disk after successful rehydration - await nativeDeleteSnapshot(); - } catch (err) { - warn('Failed to rehydrate queue from disk:', err); + private async _flushPendingDiskWrites(): Promise { + while (this.pendingDiskEvents.length > 0) { + const events = this.pendingDiskEvents; + this.pendingDiskEvents = []; + try { + await this.enqueueDiskMutation(() => this._doFlushEventsToDisk(events)); + } catch (err) { + this.pendingDiskEvents = events.concat(this.pendingDiskEvents); + throw err; + } } } - /** - * Flush current in-memory queue state to disk. - * Serialized: concurrent calls coalesce into one write. - */ - async flushToDisk(): Promise { - if (this.flushInFlight) return this.flushInFlight; - - this.flushInFlight = this._doFlushToDisk().finally(() => { - this.flushInFlight = null; - }); - return this.flushInFlight; + private appendPendingDiskEvents(events: EventPayload[]): void { + this.pendingDiskEvents.push(...events); + if (this.pendingDiskEvents.length > this.maxDiskEvents) { + const dropCount = this.pendingDiskEvents.length - this.maxDiskEvents; + this.pendingDiskEvents = this.pendingDiskEvents.slice(dropCount); + warn( + `Pending disk buffer cap reached — dropped ${dropCount} oldest events` + ); + } } - private async _doFlushToDisk(): Promise { + private async _doFlushEventsToDisk(events: EventPayload[]): Promise { try { - const queue = this.dispatcher.getQueueRef(); + const existing = await this._readExistingEvents(); + let combined = existing.concat(events); + + if (combined.length > this.maxDiskEvents) { + const dropCount = combined.length - this.maxDiskEvents; + combined = combined.slice(dropCount); + warn(`Disk store cap reached — dropped ${dropCount} oldest events`); + } - if (queue.length === 0) { - log('Queue empty — deleting snapshot'); + if (combined.length === 0) { await nativeDeleteSnapshot(); + this._hasDiskData = false; return; } const snapshot: QueueSnapshot = { version: SNAPSHOT_VERSION, - events: [...queue], // shallow copy to avoid mutation during async write + events: combined, }; - - const data = JSON.stringify(snapshot); - await writeSnapshot(data); - log(`Queue snapshot written to disk (${queue.length} events)`); + await writeSnapshot(JSON.stringify(snapshot)); + this._hasDiskData = true; + log( + `Memory queue flushed to disk: ${events.length} events, ${combined.length} total on disk` + ); } catch (err) { - warn('Failed to flush queue to disk:', err); + warn('Failed to flush events to disk:', err); + throw err; } } @@ -153,17 +229,249 @@ export class PersistentEventQueue { } /** - * Delete the disk snapshot (used during reset). + * Drain disk events directly to network in batches. Does NOT load events + * into the memory queue. Called on offline→online transition and after + * successful online flushes. + * + * Response handling: + * - 2xx: advances and deletes sent events; restores batch size after 413 + * - 413: halves batch size, retries. Drops at batchSize=1 + * - 5xx/408/429: stops, writes remainder back, retries on next online transition + * - 401/403/404: fatal, deletes disk store + * - Other 4xx: drops batch, continues + */ + async drainDiskToNetwork(dispatcher: Dispatcher): Promise { + if (this.drainInFlight) return this.drainInFlight; + + this.drainInFlight = (async () => { + try { + await this.flushPendingDiskWrites(); + } catch (err) { + warn('Disk drain continuing after pending write failure:', err); + } + await this.enqueueDiskMutation(() => + this._doDrainDiskToNetwork(dispatcher) + ); + })().finally(() => { + this.drainInFlight = null; + }); + return this.drainInFlight; + } + + private async _doDrainDiskToNetwork(dispatcher: Dispatcher): Promise { + let batchSize = DRAIN_BATCH_SIZE; + + try { + while (true) { + const raw = await readSnapshot(); + if (!raw) { + this._hasDiskData = false; + return; + } + + let snapshot: QueueSnapshot; + try { + snapshot = JSON.parse(raw); + } catch { + warn('Queue snapshot is corrupt during drain — deleting'); + await nativeDeleteSnapshot(); + this._hasDiskData = false; + return; + } + + if (!Array.isArray(snapshot.events) || snapshot.events.length === 0) { + await nativeDeleteSnapshot(); + this._hasDiskData = false; + return; + } + + const events = filterExpired(snapshot.events); + const droppedExpired = snapshot.events.length - events.length; + if (droppedExpired > 0) { + log( + `Drain TTL filter dropped ${droppedExpired} event(s) older than 7 days` + ); + } + + if (events.length === 0) { + log('All disk events expired — deleting'); + await nativeDeleteSnapshot(); + this._hasDiskData = false; + return; + } + + const n = Math.min(batchSize, events.length); + const batch = events.slice(0, n); + const remaining = events.slice(n); + + const response = await dispatcher.sendBatchDirect(batch); + + if (!response) { + log( + `Disk drain paused — network error, ${events.length} event(s) remain on disk` + ); + if (droppedExpired > 0) { + await writeSnapshot( + JSON.stringify({ version: SNAPSHOT_VERSION, events }) + ); + } + return; + } + + const category = categorizeResponse(response.statusCode); + + switch (category) { + case ResponseCategory.SUCCESS: { + if (batchSize < DRAIN_BATCH_SIZE) { + batchSize = Math.min(batchSize * 2, DRAIN_BATCH_SIZE); + } + + if (remaining.length === 0) { + await nativeDeleteSnapshot(); + this._hasDiskData = false; + log('Disk store drain complete'); + } else { + await writeSnapshot( + JSON.stringify({ + version: SNAPSHOT_VERSION, + events: remaining, + }) + ); + log( + `Disk drain batch sent (${batch.length}), ${remaining.length} remaining on disk` + ); + } + break; + } + + case ResponseCategory.PAYLOAD_TOO_LARGE: { + if (batchSize > 1) { + batchSize = Math.max(1, Math.floor(batchSize / 2)); + warn(`Disk drain: 413 — halving batch size to ${batchSize}`); + if (droppedExpired > 0) { + await writeSnapshot( + JSON.stringify({ version: SNAPSHOT_VERSION, events }) + ); + } + } else { + const ids = (batch as any[]) + .map((e) => (e as any).messageId) + .join(','); + warn( + `Disk drain: dropping oversize event(s) at batchSize=1; messageIds=${ids}` + ); + if (remaining.length === 0) { + await nativeDeleteSnapshot(); + this._hasDiskData = false; + } else { + await writeSnapshot( + JSON.stringify({ + version: SNAPSHOT_VERSION, + events: remaining, + }) + ); + } + } + break; + } + + case ResponseCategory.SERVER_ERROR: + case ResponseCategory.RATE_LIMITED: { + warn( + `Disk drain paused — ${response.statusCode}, ${events.length} event(s) remain on disk` + ); + if (droppedExpired > 0) { + await writeSnapshot( + JSON.stringify({ version: SNAPSHOT_VERSION, events }) + ); + } + return; + } + + case ResponseCategory.FATAL_CONFIG: { + warn( + `Disk drain: fatal config error ${response.statusCode} — deleting disk store` + ); + await nativeDeleteSnapshot(); + this._hasDiskData = false; + // Coordinate shutdown with the main dispatcher. Previously the + // drain path silently swallowed 401/403/404 while the main flush + // path disabled the client — inconsistent behavior. + dispatcher.handleFatalConfig(response.statusCode); + return; + } + + case ResponseCategory.CLIENT_ERROR: { + warn( + `Disk drain: dropping batch due to client error ${response.statusCode}` + ); + if (remaining.length === 0) { + await nativeDeleteSnapshot(); + this._hasDiskData = false; + } else { + await writeSnapshot( + JSON.stringify({ + version: SNAPSHOT_VERSION, + events: remaining, + }) + ); + } + break; + } + } + } + } catch (err) { + warn('Failed to drain disk to network:', err); + } + } + + /** + * Delete the disk store (used during reset). */ async deleteSnapshot(): Promise { - await nativeDeleteSnapshot(); + if (this.pendingFlushInFlight) { + try { + await this.pendingFlushInFlight; + } catch (err) { + warn('Reset continuing after pending disk write failure:', err); + } + } + await this.waitForDiskIdle(); + this.pendingDiskEvents = []; + await this.enqueueDiskMutation(async () => { + await nativeDeleteSnapshot(); + this._hasDiskData = false; + }); + } + + private async _readExistingEvents(): Promise { + const raw = await readSnapshot(); + if (!raw) return []; + try { + const snapshot: QueueSnapshot = JSON.parse(raw); + if ( + snapshot.version === SNAPSHOT_VERSION && + Array.isArray(snapshot.events) + ) { + return snapshot.events; + } + return []; + } catch { + warn('Existing disk snapshot is corrupt JSON — overwriting'); + return []; + } } } -/** - * Reset the rehydration guard. Exposed ONLY for testing. - * @internal - */ -export function _resetRehydrationGuard(): void { - hasRehydrated = false; +function filterExpired(events: EventPayload[]): EventPayload[] { + const cutoff = Date.now() - DEFAULT_EVENT_TTL_MS; + return events.filter((e) => { + const ts = (e as any).timestamp; + if (!ts) return true; + const eventTime = new Date(ts).getTime(); + // Unparseable timestamp — keep it. Conservative: better to send than to + // silently drop an event whose age we can't verify. + if (isNaN(eventTime)) return true; + return eventTime > cutoff; + }); } diff --git a/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts b/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts index 24ae688..9ea25c8 100644 --- a/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts +++ b/src/analytics/persistence/__tests__/NativeQueueStorage.test.ts @@ -1,5 +1,6 @@ import { NativeModules } from 'react-native'; import { + exists, readSnapshot, writeSnapshot, deleteSnapshot, @@ -7,6 +8,7 @@ import { function mockNativeStorage() { NativeModules.MetaRouterQueueStorage = { + exists: jest.fn().mockResolvedValue(false), readSnapshot: jest.fn().mockResolvedValue(null), writeSnapshot: jest.fn().mockResolvedValue(undefined), deleteSnapshot: jest.fn().mockResolvedValue(undefined), @@ -24,6 +26,14 @@ describe('NativeQueueStorage', () => { NativeModules.MetaRouterQueueStorage = undefined as any; }); + it('exists delegates to native module', async () => { + const mock = mockNativeStorage(); + mock.exists.mockResolvedValue(true); + const result = await exists(); + expect(result).toBe(true); + expect(mock.exists).toHaveBeenCalled(); + }); + it('readSnapshot returns null when native module returns null', async () => { const mock = mockNativeStorage(); mock.readSnapshot.mockResolvedValue(null); @@ -60,10 +70,11 @@ describe('NativeQueueStorage', () => { expect(result).toBeNull(); }); - it('writeSnapshot is a no-op if native module is missing', async () => { + it('writeSnapshot rejects if native module is missing', async () => { NativeModules.MetaRouterQueueStorage = undefined as any; - // Should not throw - await writeSnapshot('data'); + await expect(writeSnapshot('data')).rejects.toThrow( + /native module is not available/ + ); }); it('deleteSnapshot is a no-op if native module is missing', async () => { @@ -72,17 +83,21 @@ describe('NativeQueueStorage', () => { await deleteSnapshot(); }); - it('readSnapshot returns null on native module error', async () => { + it('readSnapshot rejects on native module error', async () => { const mock = mockNativeStorage(); mock.readSnapshot.mockRejectedValue(new Error('disk error')); - const result = await readSnapshot(); - expect(result).toBeNull(); + await expect(readSnapshot()).rejects.toThrow('disk error'); }); - it('writeSnapshot swallows native module error', async () => { + it('writeSnapshot rejects on native module error', async () => { const mock = mockNativeStorage(); mock.writeSnapshot.mockRejectedValue(new Error('disk error')); - // Should not throw - await writeSnapshot('data'); + await expect(writeSnapshot('data')).rejects.toThrow('disk error'); + }); + + it('deleteSnapshot rejects on native module error', async () => { + const mock = mockNativeStorage(); + mock.deleteSnapshot.mockRejectedValue(new Error('disk error')); + await expect(deleteSnapshot()).rejects.toThrow('disk error'); }); }); diff --git a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts index 6a05c57..1470005 100644 --- a/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts +++ b/src/analytics/persistence/__tests__/PersistentEventQueue.test.ts @@ -2,10 +2,10 @@ import { NativeModules } from 'react-native'; import Dispatcher from '../../dispatcher'; import CircuitBreaker from '../../utils/circuitBreaker'; -// Helpers function mockNativeStorage() { const store: { data: string | null } = { data: null }; NativeModules.MetaRouterQueueStorage = { + exists: jest.fn(async () => store.data !== null), readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { store.data = data; @@ -19,10 +19,14 @@ function mockNativeStorage() { function createDispatcher(overrides?: Partial) { return new Dispatcher({ - maxQueueBytes: 5 * 1024 * 1024, // 5MB + maxEventCount: 2000, + maxQueueBytes: 5 * 1024 * 1024, autoFlushThreshold: 9999, maxBatchSize: 100, flushIntervalSeconds: 3600, + baseRetryDelayMs: 1000, + maxRetryDelayMs: 8000, + isPersistenceEnabled: () => true, isNetworkAvailable: () => true, endpoint: (p: string) => `https://example.com${p}`, fetchWithTimeout: jest.fn(async () => ({ ok: true, status: 200 }) as any), @@ -47,576 +51,824 @@ function createDispatcher(overrides?: Partial) { describe('PersistentEventQueue', () => { beforeEach(() => { jest.clearAllMocks(); - // Reset the rehydration guard before each test - const { _resetRehydrationGuard } = require('../PersistentEventQueue'); - _resetRehydrationGuard(); }); afterEach(() => { NativeModules.MetaRouterQueueStorage = undefined as any; }); - describe('rehydrate', () => { - it('loads events from disk on first call and prepends to queue', async () => { + describe('checkForPersistedEvents', () => { + it('sets hasDiskData=true when the snapshot file exists', async () => { const { store } = mockNativeStorage(); - const events = [ - { type: 'track', event: 'disk1', messageId: '1' }, - { type: 'track', event: 'disk2', messageId: '2' }, - ]; - store.data = JSON.stringify({ version: 1, events }); + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + const result = await pq.checkForPersistedEvents(); + expect(result).toBe(true); + expect(pq.hasDiskData).toBe(true); + // Memory queue stays empty — no load on boot. + expect(dispatcher.getQueueRef().length).toBe(0); + }); + + it('sets hasDiskData=false when there is no snapshot', async () => { + mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); - expect(dispatcher.getQueueRef().length).toBe(2); - expect((dispatcher.getQueueRef()[0] as any).event).toBe('disk1'); + const result = await pq.checkForPersistedEvents(); + expect(result).toBe(false); + expect(pq.hasDiskData).toBe(false); }); - it('does not rehydrate a second time', async () => { + it('does not read the snapshot file (cheap exists-only check)', async () => { const { store, mock } = mockNativeStorage(); store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'disk1' }], + events: [{ type: 'track', event: 'e1' }], }); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - await pq.rehydrate(); + await pq.checkForPersistedEvents(); - // readSnapshot should only be called once - expect(mock.readSnapshot).toHaveBeenCalledTimes(1); - expect(dispatcher.getQueueRef().length).toBe(1); + expect(mock.exists).toHaveBeenCalled(); + expect(mock.readSnapshot).not.toHaveBeenCalled(); }); - it('enforces capacity cap on rehydrated events (drops oldest)', async () => { - const { store } = mockNativeStorage(); - // Use uniform-size events so byte cap maps to exact count - const events = Array.from({ length: 2500 }, (_, i) => ({ + it('returns false if the exists check throws', async () => { + mockNativeStorage(); + ( + NativeModules.MetaRouterQueueStorage as any + ).exists.mockRejectedValueOnce(new Error('boom')); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + const result = await pq.checkForPersistedEvents(); + expect(result).toBe(false); + expect(pq.hasDiskData).toBe(false); + }); + }); + + describe('flushToDisk', () => { + it('drains memory queue and appends to disk as versioned JSON', async () => { + const { mock, store } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + dispatcher.enqueue({ type: 'track', - event: 'evt', - properties: { id: String(i).padStart(5, '0') }, - })); - const eventSize = JSON.stringify(events[0]).length; - store.data = JSON.stringify({ version: 1, events }); + event: 'mem1', + messageId: 'a', + } as any); + dispatcher.enqueue({ + type: 'track', + event: 'mem2', + messageId: 'b', + } as any); - const dispatcher = createDispatcher({ maxQueueBytes: eventSize * 2000 }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.flushToDisk(); - expect(dispatcher.getQueueRef().length).toBe(2000); - // Should keep newest: events 500-2499 - expect((dispatcher.getQueueRef()[0] as any).properties.id).toBe('00500'); + expect(mock.writeSnapshot).toHaveBeenCalledTimes(1); + const written = JSON.parse(store.data!); + expect(written.version).toBe(1); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('mem1'); + expect(dispatcher.getQueueRef().length).toBe(0); + expect(pq.hasDiskData).toBe(true); }); - it('skips rehydration if snapshot has unknown version', async () => { + it('is a no-op when memory queue is empty', async () => { + const { mock } = mockNativeStorage(); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.flushToDisk(); + + expect(mock.writeSnapshot).not.toHaveBeenCalled(); + expect(mock.deleteSnapshot).not.toHaveBeenCalled(); + }); + + it('merges memory events with existing disk events', async () => { const { store } = mockNativeStorage(); + store.data = JSON.stringify({ - version: 99, - events: [{ type: 'track', event: 'x' }], + version: 1, + events: [{ type: 'track', event: 'disk1' }], }); const dispatcher = createDispatcher(); + dispatcher.enqueue({ type: 'track', event: 'mem1' } as any); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.flushToDisk(); - expect(dispatcher.getQueueRef().length).toBe(0); + const written = JSON.parse(store.data!); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('disk1'); + expect(written.events[1].event).toBe('mem1'); }); - it('handles corrupt JSON gracefully', async () => { - const { store } = mockNativeStorage(); - store.data = '{not valid json'; + it('restores memory events and rejects when native write fails', async () => { + const { mock } = mockNativeStorage(); + mock.writeSnapshot.mockRejectedValueOnce(new Error('disk full')); const dispatcher = createDispatcher(); + dispatcher.enqueue({ type: 'track', event: 'mem1' } as any); + dispatcher.enqueue({ type: 'track', event: 'mem2' } as any); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - - expect(dispatcher.getQueueRef().length).toBe(0); + await expect(pq.flushToDisk()).rejects.toThrow('disk full'); + expect(dispatcher.getQueueRef().map((e: any) => e.event)).toEqual([ + 'mem1', + 'mem2', + ]); + expect(pq.hasDiskData).toBe(false); }); + }); - it('handles null snapshot (no file on disk)', async () => { + describe('shouldFlushToDisk', () => { + it('returns false when below byte threshold', () => { mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + for (let i = 0; i < 10; i++) { + dispatcher.enqueue({ type: 'track', event: `e${i}` } as any); + } - expect(dispatcher.getQueueRef().length).toBe(0); + expect(pq.shouldFlushToDisk()).toBe(false); }); - it('deletes snapshot after successful rehydration', async () => { - const { store, mock } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 1, - events: [{ type: 'track', event: 'x' }], - }); + it('returns true when byte size exceeds threshold', () => { + mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + const bigProps: Record = {}; + for (let i = 0; i < 20; i++) { + bigProps[`key${i}`] = 'x'.repeat(100); + } - expect(mock.deleteSnapshot).toHaveBeenCalled(); + for (let i = 0; i < 1000; i++) { + dispatcher.enqueue({ + type: 'track', + event: `e${i}`, + properties: bigProps, + } as any); + } + + expect(pq.shouldFlushToDisk()).toBe(true); }); + }); - it('deletes snapshot when version is unknown', async () => { - const { store, mock } = mockNativeStorage(); - store.data = JSON.stringify({ - version: 99, - events: [{ type: 'track', event: 'x' }], - }); + describe('deleteSnapshot', () => { + it('delegates to native module and clears hasDiskData', async () => { + const { mock } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + // Force hasDiskData=true by flushing something first + dispatcher.enqueue({ type: 'track', event: 'e1' } as any); + await pq.flushToDisk(); + expect(pq.hasDiskData).toBe(true); + + await pq.deleteSnapshot(); expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(dispatcher.getQueueRef().length).toBe(0); + expect(pq.hasDiskData).toBe(false); }); + }); - it('deletes snapshot when JSON is corrupt', async () => { - const { store, mock } = mockNativeStorage(); - store.data = '{not valid json'; + describe('flushEventsToDisk', () => { + it('writes events to disk store', async () => { + const { store } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.flushEventsToDisk([ + { type: 'track', event: 'o1' }, + { type: 'track', event: 'o2' }, + ]); - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(dispatcher.getQueueRef().length).toBe(0); + const written = JSON.parse(store.data!); + expect(written.version).toBe(1); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('o1'); }); - }); - describe('resetRehydrationGuard', () => { - it('allows rehydration again after reset', async () => { - const { store, mock } = mockNativeStorage(); + it('merges with existing events on disk', async () => { + const { store } = mockNativeStorage(); + store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'e1' }], + events: [{ type: 'track', event: 'existing1' }], }); const dispatcher = createDispatcher(); - const { - PersistentEventQueue, - _resetRehydrationGuard, - } = require('../PersistentEventQueue'); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - expect(dispatcher.getQueueRef().length).toBe(1); + await pq.flushEventsToDisk([{ type: 'track', event: 'new1' }]); + + const written = JSON.parse(store.data!); + expect(written.events.length).toBe(2); + expect(written.events[0].event).toBe('existing1'); + expect(written.events[1].event).toBe('new1'); + }); + + it('enforces disk cap when merging', async () => { + const { store } = mockNativeStorage(); - // Reset guard and provide new snapshot - _resetRehydrationGuard(); store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'e2' }], + events: Array.from({ length: 4 }, (_, i) => ({ + type: 'track', + event: `old${i}`, + })), + }); + + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher, { + maxDiskEvents: 5, }); - const pq2 = new PersistentEventQueue(dispatcher); - await pq2.rehydrate(); + await pq.flushEventsToDisk([ + { type: 'track', event: 'new1' }, + { type: 'track', event: 'new2' }, + ]); - // Should have rehydrated the second time - expect(mock.readSnapshot).toHaveBeenCalledTimes(2); + const written = JSON.parse(store.data!); + expect(written.events.length).toBe(5); + expect(written.events[0].event).toBe('old1'); }); - }); - describe('flushToDisk', () => { - it('writes current queue state to disk as versioned JSON', async () => { + it('is a no-op when events array is empty', async () => { const { mock } = mockNativeStorage(); const dispatcher = createDispatcher(); - dispatcher.enqueue({ - type: 'track', - event: 'mem1', - messageId: 'a', - } as any); - dispatcher.enqueue({ - type: 'track', - event: 'mem2', - messageId: 'b', - } as any); - const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.flushToDisk(); + await pq.flushEventsToDisk([]); - expect(mock.writeSnapshot).toHaveBeenCalledTimes(1); - const written = JSON.parse(mock.writeSnapshot.mock.calls[0][0]); - expect(written.version).toBe(1); - expect(written.events.length).toBe(2); - expect(written.events[0].event).toBe('mem1'); + expect(mock.writeSnapshot).not.toHaveBeenCalled(); }); - it('deletes snapshot if queue is empty', async () => { - const { mock } = mockNativeStorage(); + it('serializes concurrent writes (no races)', async () => { + const { mock, store } = mockNativeStorage(); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.flushToDisk(); + const p1 = pq.flushEventsToDisk([{ type: 'track', event: 'a' }]); + const p2 = pq.flushEventsToDisk([{ type: 'track', event: 'b' }]); - expect(mock.deleteSnapshot).toHaveBeenCalled(); - expect(mock.writeSnapshot).not.toHaveBeenCalled(); + await Promise.all([p1, p2]); + + expect(mock.writeSnapshot).toHaveBeenCalledTimes(2); + const written = JSON.parse(store.data!); + expect(written.events.map((e: any) => e.event)).toEqual(['a', 'b']); }); - it('serializes flushToDisk calls (no concurrent writes)', async () => { + it('rejects when native write fails', async () => { const { mock } = mockNativeStorage(); - - let writeResolve: (() => void) | null = null; - mock.writeSnapshot = jest.fn( - () => - new Promise((resolve) => { - writeResolve = resolve; - }) - ); + mock.writeSnapshot.mockRejectedValueOnce(new Error('disk full')); const dispatcher = createDispatcher(); - dispatcher.enqueue({ type: 'track', event: 'e1' } as any); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + await expect( + pq.flushEventsToDisk([{ type: 'track', event: 'a' }]) + ).rejects.toThrow('disk full'); + expect(pq.hasDiskData).toBe(false); + }); + + it('keeps buffered overflow events pending when write fails and retries later', async () => { + const { mock, store } = mockNativeStorage(); + mock.writeSnapshot + .mockRejectedValueOnce(new Error('disk full')) + .mockImplementation(async (data: string) => { + store.data = data; + }); + + const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - const p1 = pq.flushToDisk(); - const p2 = pq.flushToDisk(); + pq.bufferEventsForDisk([{ type: 'track', event: 'overflowed' }]); - // Only one write should be in flight - expect(mock.writeSnapshot).toHaveBeenCalledTimes(1); + await expect(pq.flushPendingDiskWrites()).rejects.toThrow('disk full'); + expect(store.data).toBeNull(); - writeResolve!(); - await p1; - await p2; + await pq.flushPendingDiskWrites(); + const written = JSON.parse(store.data!); + expect(written.events.map((e: any) => e.event)).toEqual(['overflowed']); }); }); - describe('shouldFlushToDisk', () => { - it('returns false when below byte threshold', () => { - mockNativeStorage(); + describe('drainDiskToNetwork', () => { + it('sends disk events directly to network without entering memory queue', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [ + { type: 'track', event: 'disk1' }, + { type: 'track', event: 'disk2' }, + ], + }); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - for (let i = 0; i < 10; i++) { - dispatcher.enqueue({ type: 'track', event: `e${i}` } as any); - } + await pq.drainDiskToNetwork(dispatcher); - expect(pq.shouldFlushToDisk()).toBe(false); + expect(dispatcher.getQueueRef().length).toBe(0); + expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalled(); }); - }); - describe('deleteSnapshot', () => { - it('delegates to native module', async () => { - const { mock } = mockNativeStorage(); + it('flushes pending overflow writes before draining disk to network', async () => { + const { store } = mockNativeStorage(); - const dispatcher = createDispatcher(); + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'disk1' }], + }); + + const sentEvents: string[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + const body = JSON.parse(init.body); + sentEvents.push(...body.batch.map((e: any) => e.event)); + return { ok: true, status: 200 } as any; + }), + }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.deleteSnapshot(); + pq.bufferEventsForDisk([{ type: 'track', event: 'overflow1' }]); + await pq.drainDiskToNetwork(dispatcher); - expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(sentEvents).toEqual(['disk1', 'overflow1']); + expect(store.data).toBeNull(); }); - }); - describe('size-based flush threshold', () => { - it('shouldFlushToDisk returns true when byte size exceeds threshold', () => { - mockNativeStorage(); + it('deletes disk file after drain and clears hasDiskData', async () => { + const { store, mock } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - // Create a large event (~2KB each, need ~1000 to reach 2MB) - const bigProps: Record = {}; - for (let i = 0; i < 20; i++) { - bigProps[`key${i}`] = 'x'.repeat(100); - } + await pq.checkForPersistedEvents(); + expect(pq.hasDiskData).toBe(true); - // Enqueue enough events to exceed 2MB - for (let i = 0; i < 1000; i++) { - dispatcher.enqueue({ + await pq.drainDiskToNetwork(dispatcher); + + expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(store.data).toBeNull(); + expect(pq.hasDiskData).toBe(false); + }); + + it('stops on 5xx server error', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: Array.from({ length: 5 }, (_, i) => ({ type: 'track', event: `e${i}`, - properties: bigProps, - } as any); - } + })), + }); - expect(pq.shouldFlushToDisk()).toBe(true); + const fetchMock = jest.fn(async () => ({ + ok: false, + status: 500, + })); + const dispatcher = createDispatcher({ + fetchWithTimeout: fetchMock, + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(store.data).not.toBeNull(); + const remaining = JSON.parse(store.data!); + expect(remaining.events.length).toBe(5); }); - }); - describe('rehydrate + enqueue ordering', () => { - it('rehydrated events appear before new events', async () => { + it('stops on 429 rate limit', async () => { const { store } = mockNativeStorage(); + store.data = JSON.stringify({ version: 1, - events: [ - { type: 'track', event: 'old1', messageId: '1' }, - { type: 'track', event: 'old2', messageId: '2' }, - ], + events: [{ type: 'track', event: 'e1' }], }); - const dispatcher = createDispatcher(); + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 429, + })), + }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.drainDiskToNetwork(dispatcher); - // Now enqueue new events - dispatcher.enqueue({ type: 'track', event: 'new1' } as any); + expect(store.data).not.toBeNull(); + }); + + it('halves batch size on 413 and retries', async () => { + const { store } = mockNativeStorage(); - const queue = dispatcher.getQueueRef(); - expect((queue[0] as any).event).toBe('old1'); - expect((queue[1] as any).event).toBe('old2'); - expect((queue[2] as any).event).toBe('new1'); + store.data = JSON.stringify({ + version: 1, + events: Array.from({ length: 200 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); + + const batchSizes: number[] = []; + let callCount = 0; + const fetchMock = jest.fn(async (_url?: string, init?: any) => { + callCount++; + const batchLen = JSON.parse(init.body).batch.length; + batchSizes.push(batchLen); + if (callCount === 1) { + return { ok: false, status: 413 }; + } + return { ok: true, status: 200 }; + }); + + const dispatcher = createDispatcher({ fetchWithTimeout: fetchMock }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(batchSizes).toEqual([100, 50, 100, 50]); + expect(store.data).toBeNull(); }); - }); - describe('TTL expiry', () => { - it('drops events older than 7 days on rehydrate', async () => { + it('drops events on 413 at batchSize=1', async () => { const { store } = mockNativeStorage(); - const now = Date.now(); - const eightDaysAgo = new Date( - now - 8 * 24 * 60 * 60 * 1000 - ).toISOString(); - const oneDayAgo = new Date(now - 1 * 24 * 60 * 60 * 1000).toISOString(); store.data = JSON.stringify({ version: 1, events: [ - { type: 'track', event: 'expired', timestamp: eightDaysAgo }, - { type: 'track', event: 'fresh', timestamp: oneDayAgo }, + { type: 'track', event: 'big_event', messageId: 'msg1' }, + { type: 'track', event: 'normal_event', messageId: 'msg2' }, ], }); - const dispatcher = createDispatcher(); + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 413, + })), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(store.data).toBeNull(); + }); + + it('deletes disk store on fatal config error (401/403/404)', async () => { + const { store, mock } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); + + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 403, + })), + }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.drainDiskToNetwork(dispatcher); - expect(dispatcher.getQueueRef().length).toBe(1); - expect((dispatcher.getQueueRef()[0] as any).event).toBe('fresh'); + expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(store.data).toBeNull(); }); - it('keeps events without a timestamp field', async () => { + it('fires onFatalConfig handler on 401/403/404 during drain', async () => { const { store } = mockNativeStorage(); + store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'no_ts' }], + events: [{ type: 'track', event: 'e1' }], }); - const dispatcher = createDispatcher(); + const onFatalConfig = jest.fn(); + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => ({ + ok: false, + status: 401, + })), + onFatalConfig, + }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.drainDiskToNetwork(dispatcher); - expect(dispatcher.getQueueRef().length).toBe(1); + expect(onFatalConfig).toHaveBeenCalledTimes(1); }); - it('discards snapshot and deletes from disk when all events are expired', async () => { - const { store, mock } = mockNativeStorage(); - const eightDaysAgo = new Date( - Date.now() - 8 * 24 * 60 * 60 * 1000 - ).toISOString(); + it('drops batch on other 4xx and continues draining', async () => { + const { store } = mockNativeStorage(); store.data = JSON.stringify({ version: 1, - events: [ - { type: 'track', event: 'old1', timestamp: eightDaysAgo }, - { type: 'track', event: 'old2', timestamp: eightDaysAgo }, - ], + events: Array.from({ length: 150 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), }); - const dispatcher = createDispatcher(); + let callCount = 0; + const fetchMock = jest.fn(async () => { + callCount++; + if (callCount === 1) { + return { ok: false, status: 400 }; + } + return { ok: true, status: 200 }; + }); + + const dispatcher = createDispatcher({ fetchWithTimeout: fetchMock }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.drainDiskToNetwork(dispatcher); - expect(dispatcher.getQueueRef().length).toBe(0); - expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(store.data).toBeNull(); }); - }); - describe('sentAt on rehydrated events', () => { - it('does not set sentAt at rehydration time (drainBatch stamps it at send time)', async () => { + it('restores batch size after success following 413', async () => { const { store } = mockNativeStorage(); - const oneDayAgo = new Date( - Date.now() - 24 * 60 * 60 * 1000 - ).toISOString(); store.data = JSON.stringify({ version: 1, - events: [{ type: 'track', event: 'e1', timestamp: oneDayAgo }], + events: Array.from({ length: 300 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), }); - const dispatcher = createDispatcher(); + const batchSizes: number[] = []; + let callCount = 0; + const fetchMock = jest.fn(async (_url?: string, init?: any) => { + callCount++; + const batchLen = JSON.parse(init.body).batch.length; + batchSizes.push(batchLen); + if (callCount === 1) return { ok: false, status: 413 }; + return { ok: true, status: 200 }; + }); + + const dispatcher = createDispatcher({ fetchWithTimeout: fetchMock }); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); + await pq.drainDiskToNetwork(dispatcher); - const rehydrated = dispatcher.getQueueRef()[0] as any; - expect(rehydrated.sentAt).toBeUndefined(); - // Original timestamp preserved - expect(rehydrated.timestamp).toBe(oneDayAgo); + expect(batchSizes[0]).toBe(100); + expect(batchSizes[1]).toBe(50); + expect(batchSizes[2]).toBe(100); }); - }); - describe('rehydratedEvents count', () => { - it('tracks the number of rehydrated events', async () => { + it('drains in batches of 100', async () => { const { store } = mockNativeStorage(); + store.data = JSON.stringify({ version: 1, - events: [ - { type: 'track', event: 'e1' }, - { type: 'track', event: 'e2' }, - { type: 'track', event: 'e3' }, - ], + events: Array.from({ length: 250 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), }); + const fetchCalls: number[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + fetchCalls.push(JSON.parse(init.body).batch.length); + return { ok: true, status: 200 } as any; + }), + }); + + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); + + await pq.drainDiskToNetwork(dispatcher); + + expect(fetchCalls).toEqual([100, 100, 50]); + expect(store.data).toBeNull(); + }); + + it('is a no-op when no disk file exists', async () => { + mockNativeStorage(); + const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - expect(pq.rehydratedEvents).toBe(0); - await pq.rehydrate(); - expect(pq.rehydratedEvents).toBe(3); + await pq.drainDiskToNetwork(dispatcher); + + expect((dispatcher as any).opts.fetchWithTimeout).not.toHaveBeenCalled(); }); - it('excludes expired events from count', async () => { + it('filters expired events during drain', async () => { const { store } = mockNativeStorage(); + const eightDaysAgo = new Date( Date.now() - 8 * 24 * 60 * 60 * 1000 ).toISOString(); const oneDayAgo = new Date( - Date.now() - 24 * 60 * 60 * 1000 + Date.now() - 1 * 24 * 60 * 60 * 1000 ).toISOString(); store.data = JSON.stringify({ version: 1, events: [ { type: 'track', event: 'expired', timestamp: eightDaysAgo }, - { type: 'track', event: 'fresh1', timestamp: oneDayAgo }, - { type: 'track', event: 'fresh2', timestamp: oneDayAgo }, + { type: 'track', event: 'fresh', timestamp: oneDayAgo }, ], }); - const dispatcher = createDispatcher(); + let sentBatch: any[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + sentBatch = JSON.parse(init.body).batch; + return { ok: true, status: 200 } as any; + }), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - expect(pq.rehydratedEvents).toBe(2); + await pq.drainDiskToNetwork(dispatcher); + + expect(sentBatch.length).toBe(1); + expect(sentBatch[0].event).toBe('fresh'); }); - it('is 0 when no snapshot exists', async () => { - mockNativeStorage(); + it('keeps events with unparseable timestamps during drain', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: [ + { type: 'track', event: 'no_ts' }, + { type: 'track', event: 'bad_ts', timestamp: 'not-a-date' }, + ], + }); + + let sentBatch: any[] = []; + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async (_url?: string, init?: any) => { + sentBatch = JSON.parse(init.body).batch; + return { ok: true, status: 200 } as any; + }), + }); - const dispatcher = createDispatcher(); const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - await pq.rehydrate(); - expect(pq.rehydratedEvents).toBe(0); + await pq.drainDiskToNetwork(dispatcher); + + expect(sentBatch.length).toBe(2); }); - }); - describe('full round-trip', () => { - it('enqueue → flushToDisk → rehydrate preserves events', async () => { - mockNativeStorage(); + it('stops on network/transport error (null response)', async () => { + const { store } = mockNativeStorage(); - const dispatcher = createDispatcher(); - const { - PersistentEventQueue, - _resetRehydrationGuard, - } = require('../PersistentEventQueue'); + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'e1' }], + }); + + const dispatcher = createDispatcher({ + fetchWithTimeout: jest.fn(async () => { + throw new Error('Network unavailable'); + }), + }); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); - // Enqueue events - for (let i = 0; i < 5; i++) { - dispatcher.enqueue({ - type: 'track', - event: `e${i}`, - properties: { i }, - } as any); - } + await pq.drainDiskToNetwork(dispatcher); - // Flush to disk - await pq.flushToDisk(); + expect(store.data).not.toBeNull(); + }); - // Simulate process restart: new dispatcher, reset guard - _resetRehydrationGuard(); - const dispatcher2 = createDispatcher(); - const pq2 = new PersistentEventQueue(dispatcher2); + it('concurrent drains coalesce into one', async () => { + const { store } = mockNativeStorage(); + + store.data = JSON.stringify({ + version: 1, + events: Array.from({ length: 3 }, (_, i) => ({ + type: 'track', + event: `e${i}`, + })), + }); - await pq2.rehydrate(); + const dispatcher = createDispatcher(); + const { PersistentEventQueue } = require('../PersistentEventQueue'); + const pq = new PersistentEventQueue(dispatcher); - expect(dispatcher2.getQueueRef().length).toBe(5); - expect((dispatcher2.getQueueRef()[0] as any).event).toBe('e0'); - expect((dispatcher2.getQueueRef()[4] as any).event).toBe('e4'); + const [a, b] = await Promise.all([ + pq.drainDiskToNetwork(dispatcher), + pq.drainDiskToNetwork(dispatcher), + ]); + expect(a).toBe(b); + expect((dispatcher as any).opts.fetchWithTimeout).toHaveBeenCalledTimes( + 1 + ); }); + }); - it('partial drain then flush only persists remaining events', async () => { + describe('full round-trip (persist → drain)', () => { + it('flushToDisk persists events, drainDiskToNetwork sends them', async () => { mockNativeStorage(); const dispatcher = createDispatcher(); - const { - PersistentEventQueue, - _resetRehydrationGuard, - } = require('../PersistentEventQueue'); + const { PersistentEventQueue } = require('../PersistentEventQueue'); const pq = new PersistentEventQueue(dispatcher); for (let i = 0; i < 5; i++) { dispatcher.enqueue({ type: 'track', event: `e${i}`, + properties: { i }, } as any); } - // Flush network drains events - await dispatcher.flush(); - // Queue should be empty after successful network flush - expect(dispatcher.getQueueRef().length).toBe(0); - - // Add 2 more events - dispatcher.enqueue({ type: 'track', event: 'late1' } as any); - dispatcher.enqueue({ type: 'track', event: 'late2' } as any); - - // Flush to disk await pq.flushToDisk(); + expect(pq.hasDiskData).toBe(true); - // Simulate restart - _resetRehydrationGuard(); const dispatcher2 = createDispatcher(); const pq2 = new PersistentEventQueue(dispatcher2); - await pq2.rehydrate(); - // Should only have the 2 late events - expect(dispatcher2.getQueueRef().length).toBe(2); - expect((dispatcher2.getQueueRef()[0] as any).event).toBe('late1'); + await pq2.checkForPersistedEvents(); + expect(pq2.hasDiskData).toBe(true); + + await pq2.drainDiskToNetwork(dispatcher2); + + // All 5 events delivered via a single batch + expect((dispatcher2 as any).opts.fetchWithTimeout).toHaveBeenCalledTimes( + 1 + ); + const body = JSON.parse( + (dispatcher2 as any).opts.fetchWithTimeout.mock.calls[0][1].body + ); + expect(body.batch.length).toBe(5); + expect(pq2.hasDiskData).toBe(false); }); }); }); diff --git a/src/analytics/persistence/__tests__/integration.test.ts b/src/analytics/persistence/__tests__/integration.test.ts index 124243c..edc9e3c 100644 --- a/src/analytics/persistence/__tests__/integration.test.ts +++ b/src/analytics/persistence/__tests__/integration.test.ts @@ -1,9 +1,9 @@ import { NativeModules, AppState } from 'react-native'; -import { _resetRehydrationGuard } from '../PersistentEventQueue'; function mockNativeStorage() { const store: { data: string | null } = { data: null }; NativeModules.MetaRouterQueueStorage = { + exists: jest.fn(async () => store.data !== null), readSnapshot: jest.fn(async () => store.data), writeSnapshot: jest.fn(async (data: string) => { store.data = data; @@ -20,7 +20,6 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { beforeEach(() => { jest.clearAllMocks(); - _resetRehydrationGuard(); handleAppStateChange = null; // Mock fetch so flush() doesn't hang on a real network call @@ -41,7 +40,7 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { (global.fetch as jest.Mock).mockRestore?.(); }); - it('rehydrates events from disk during init', async () => { + it('drains on-disk events directly to network during init (no memory rehydrate)', async () => { const { store } = mockNativeStorage(); store.data = JSON.stringify({ version: 1, @@ -66,7 +65,10 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { }); await client.init(); - // Rehydrated events should have been flushed immediately on init + // Allow async drain to run + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Disk events should have been sent via drainDiskToNetwork → fetch expect(global.fetch).toHaveBeenCalled(); const body = JSON.parse((global.fetch as jest.Mock).mock.calls[0][1].body); expect(body.batch).toEqual( @@ -76,10 +78,40 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { ); }); + it('does not load disk events into the memory queue on init (cheap exists-only check)', async () => { + const { store, mock } = mockNativeStorage(); + store.data = JSON.stringify({ + version: 1, + events: [{ type: 'track', event: 'persisted' }], + }); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }, + { networkMonitor: monitor } + ); + await client.init(); + + // Offline, so no drain runs. readSnapshot should not have fired. + expect(mock.exists).toHaveBeenCalled(); + expect(mock.readSnapshot).not.toHaveBeenCalled(); + // Memory queue remains empty — disk events will drain when online. + const debug = await client.getDebugInfo(); + expect(debug.queueLength).toBe(0); + expect(debug.hasDiskData).toBe(true); + }); + it('flushes to disk when app goes to background and network fails', async () => { const { mock } = mockNativeStorage(); - // Simulate network failure so events remain in queue for persistence (global.fetch as jest.Mock).mockImplementation(() => Promise.reject(new Error('Network unavailable')) ); @@ -93,17 +125,15 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { }); await client.init(); - // Track some events client.track('event1', { key: 'value' }); client.track('event2', { key: 'value' }); - // Simulate app going to background — flush fails, events persisted to disk await handleAppStateChange!('background'); expect(mock.writeSnapshot).toHaveBeenCalled(); }); - it('skips disk write when network flush succeeds on background', async () => { + it('leaves disk untouched when network flush succeeds on background with empty memory', async () => { const { mock } = mockNativeStorage(); const { @@ -115,16 +145,13 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { }); await client.init(); - // Track some events client.track('event1', { key: 'value' }); client.track('event2', { key: 'value' }); - // Simulate app going to background — flush succeeds, queue empty await handleAppStateChange!('background'); - // Queue was drained by network flush, so deleteSnapshot is called (empty queue) expect(mock.writeSnapshot).not.toHaveBeenCalled(); - expect(mock.deleteSnapshot).toHaveBeenCalled(); + expect(mock.deleteSnapshot).not.toHaveBeenCalled(); }); it('deletes snapshot on reset', async () => { @@ -143,4 +170,97 @@ describe('MetaRouterAnalyticsClient + persistence integration', () => { expect(mock.deleteSnapshot).toHaveBeenCalled(); }); + + it('capacity overflow writes events to disk', async () => { + const { mock } = mockNativeStorage(); + + (global.fetch as jest.Mock).mockImplementation(() => + Promise.reject(new Error('Network unavailable')) + ); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + maxQueueEvents: 10, + maxDiskEvents: 100, + }, + { networkMonitor: monitor } + ); + await client.init(); + + for (let i = 0; i < 50; i++) { + client.track(`event_${i}`, { idx: i }); + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mock.writeSnapshot).toHaveBeenCalled(); + }); + + it('events enqueue successfully while offline (no errors, no HTTP attempts)', async () => { + mockNativeStorage(); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }, + { networkMonitor: monitor } + ); + await client.init(); + + client.track('offline_event_1'); + client.track('offline_event_2'); + + await client.flush(); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it('offline flush sends events to disk, online drains them', async () => { + const { mock } = mockNativeStorage(); + + const { + MetaRouterAnalyticsClient, + } = require('../../MetaRouterAnalyticsClient'); + const { StubNetworkMonitor } = require('../../utils/networkMonitor'); + const monitor = new StubNetworkMonitor('disconnected'); + + const client = new MetaRouterAnalyticsClient( + { + writeKey: 'test-key', + ingestionHost: 'https://example.com', + }, + { networkMonitor: monitor } + ); + await client.init(); + + client.track('offline_1'); + client.track('offline_2'); + + await client.flush(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(mock.writeSnapshot).toHaveBeenCalled(); + expect(global.fetch).not.toHaveBeenCalled(); + + monitor.simulate('connected'); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(global.fetch).toHaveBeenCalled(); + }); }); diff --git a/src/analytics/persistence/types.ts b/src/analytics/persistence/types.ts index 9476758..9f398d4 100644 --- a/src/analytics/persistence/types.ts +++ b/src/analytics/persistence/types.ts @@ -20,3 +20,6 @@ export const FLUSH_THRESHOLD_BYTES = 2 * 1024 * 1024; /** Flush-to-disk threshold: event count */ export const FLUSH_THRESHOLD_COUNT = 500; + +/** Default max events stored on disk (crash safety + extended offline overflow) */ +export const DEFAULT_MAX_DISK_EVENTS = 10_000; diff --git a/src/analytics/types.ts b/src/analytics/types.ts index 87f7687..fbbbed0 100644 --- a/src/analytics/types.ts +++ b/src/analytics/types.ts @@ -31,10 +31,15 @@ export interface InitOptions { ingestionHost: string; flushIntervalSeconds?: number; debug?: boolean; - /** Max bytes (UTF-8) held in memory queue; oldest are dropped once cap is hit (default: 5MB) */ - maxQueueBytes?: number; - /** Max events stored on disk during extended offline periods (default: 10000) */ - maxOfflineDiskEvents?: number; + /** Max events held in the in-memory queue (default: 2000). Clamped to >= 1. */ + maxQueueEvents?: number; + /** + * Max events stored on disk for crash-safety + extended offline periods + * (default: 10000). Must be >= 0. When set to 0, disk persistence is + * disabled and the SDK runs as a pure in-memory pipeline (events are lost + * on app kill — documented tradeoff). + */ + maxDiskEvents?: number; } export interface AnalyticsInterface { diff --git a/src/analytics/utils/debouncedNetworkMonitor.test.ts b/src/analytics/utils/debouncedNetworkMonitor.test.ts new file mode 100644 index 0000000..2c5b3fe --- /dev/null +++ b/src/analytics/utils/debouncedNetworkMonitor.test.ts @@ -0,0 +1,139 @@ +import { DebouncedNetworkMonitor } from './debouncedNetworkMonitor'; +import { StubNetworkMonitor } from './networkMonitor'; + +describe('DebouncedNetworkMonitor', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('inherits initial status from the inner monitor', () => { + const inner = new StubNetworkMonitor('connected'); + const debounced = new DebouncedNetworkMonitor(inner); + expect(debounced.currentStatus).toBe('connected'); + + const inner2 = new StubNetworkMonitor('disconnected'); + const debounced2 = new DebouncedNetworkMonitor(inner2); + expect(debounced2.currentStatus).toBe('disconnected'); + }); + + it('fires offline transitions immediately', () => { + const inner = new StubNetworkMonitor('connected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('disconnected'); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith('disconnected'); + expect(debounced.currentStatus).toBe('disconnected'); + }); + + it('debounces online transitions by 2s of stable connectivity', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + + // Not yet fired — still within debounce window. + expect(handler).not.toHaveBeenCalled(); + expect(debounced.currentStatus).toBe('disconnected'); + + jest.advanceTimersByTime(1999); + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(1); + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith('connected'); + expect(debounced.currentStatus).toBe('connected'); + }); + + it('cancels pending online fire if offline arrives during the debounce window', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + jest.advanceTimersByTime(1500); + inner.simulate('disconnected'); + + // No online transition ever fired; no offline transition either + // (we were already at 'disconnected' from the debounced perspective). + jest.advanceTimersByTime(5000); + expect(handler).not.toHaveBeenCalled(); + expect(debounced.currentStatus).toBe('disconnected'); + }); + + it('resets the debounce timer on repeated online signals', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + jest.advanceTimersByTime(1000); + // Another online signal before timer fires — should restart the window. + (inner as any)._currentStatus = 'disconnected'; + inner.simulate('connected'); + jest.advanceTimersByTime(1999); + expect(handler).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1); + expect(handler).toHaveBeenCalledTimes(1); + }); + + it('fires online after offline when debounce elapses', () => { + const inner = new StubNetworkMonitor('connected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('disconnected'); + expect(handler).toHaveBeenNthCalledWith(1, 'disconnected'); + + inner.simulate('connected'); + jest.advanceTimersByTime(2000); + expect(handler).toHaveBeenNthCalledWith(2, 'connected'); + }); + + it('stop() cancels pending online timer and unsubscribes from inner', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + debounced.stop(); + + jest.advanceTimersByTime(5000); + expect(handler).not.toHaveBeenCalled(); + + // Further simulations on inner no longer propagate + inner.simulate('disconnected'); + inner.simulate('connected'); + jest.advanceTimersByTime(5000); + expect(handler).not.toHaveBeenCalled(); + }); + + it('does not fire when inner flips offline before the fire callback runs', () => { + const inner = new StubNetworkMonitor('disconnected'); + const debounced = new DebouncedNetworkMonitor(inner); + const handler = jest.fn(); + debounced.onStatusChange(handler); + + inner.simulate('connected'); + // Flip inner offline silently (simulate a race: timer is scheduled but + // inner.currentStatus changes before the callback runs). + (inner as any)._currentStatus = 'disconnected'; + + jest.advanceTimersByTime(2000); + expect(handler).not.toHaveBeenCalled(); + expect(debounced.currentStatus).toBe('disconnected'); + }); +}); diff --git a/src/analytics/utils/debouncedNetworkMonitor.ts b/src/analytics/utils/debouncedNetworkMonitor.ts new file mode 100644 index 0000000..5926658 --- /dev/null +++ b/src/analytics/utils/debouncedNetworkMonitor.ts @@ -0,0 +1,76 @@ +import type { NetworkReachability, NetworkStatus } from './networkMonitor'; + +/** + * Wraps a {@link NetworkReachability} with asymmetric debouncing: + * - **offline** transitions fire immediately (pause HTTP ASAP — no point + * burning retries on a down network). + * - **online** transitions wait for {@link ONLINE_DEBOUNCE_MS} of stable + * connectivity before firing, so flaky WiFi/cellular handoffs don't + * trigger spurious flush attempts that immediately fail. + * + * A pending online debounce is cancelled if offline arrives during the wait. + */ +export class DebouncedNetworkMonitor implements NetworkReachability { + static readonly ONLINE_DEBOUNCE_MS = 2000; + + private readonly inner: NetworkReachability; + private handler: ((status: NetworkStatus) => void) | null = null; + private unsubscribeInner: (() => void) | null = null; + private onlineTimer: ReturnType | null = null; + private _currentStatus: NetworkStatus; + + constructor(inner: NetworkReachability) { + this.inner = inner; + this._currentStatus = inner.currentStatus; + + this.unsubscribeInner = inner.onStatusChange((rawStatus) => { + if (rawStatus === 'disconnected') { + this.clearOnlineTimer(); + if (this._currentStatus !== 'disconnected') { + this._currentStatus = 'disconnected'; + this.handler?.('disconnected'); + } + return; + } + + // rawStatus === 'connected': start/reset the debounce window + this.clearOnlineTimer(); + if (this._currentStatus === 'connected') return; + + this.onlineTimer = setTimeout(() => { + this.onlineTimer = null; + // Guard against racing with a late offline transition. + if (this.inner.currentStatus !== 'connected') return; + if (this._currentStatus !== 'connected') { + this._currentStatus = 'connected'; + this.handler?.('connected'); + } + }, DebouncedNetworkMonitor.ONLINE_DEBOUNCE_MS); + }); + } + + get currentStatus(): NetworkStatus { + return this._currentStatus; + } + + onStatusChange(handler: (status: NetworkStatus) => void): () => void { + this.handler = handler; + return () => { + this.handler = null; + }; + } + + stop(): void { + this.clearOnlineTimer(); + this.unsubscribeInner?.(); + this.unsubscribeInner = null; + this.handler = null; + } + + private clearOnlineTimer(): void { + if (this.onlineTimer) { + clearTimeout(this.onlineTimer); + this.onlineTimer = null; + } + } +} diff --git a/src/analytics/utils/responseCategory.ts b/src/analytics/utils/responseCategory.ts new file mode 100644 index 0000000..927b625 --- /dev/null +++ b/src/analytics/utils/responseCategory.ts @@ -0,0 +1,23 @@ +/** + * Shared classification of HTTP status codes used by both + * the dispatcher and the disk drain. + */ +export enum ResponseCategory { + SUCCESS = 'SUCCESS', + SERVER_ERROR = 'SERVER_ERROR', + RATE_LIMITED = 'RATE_LIMITED', + PAYLOAD_TOO_LARGE = 'PAYLOAD_TOO_LARGE', + FATAL_CONFIG = 'FATAL_CONFIG', + CLIENT_ERROR = 'CLIENT_ERROR', +} + +export function categorizeResponse(statusCode: number): ResponseCategory { + if (statusCode >= 200 && statusCode < 300) return ResponseCategory.SUCCESS; + if (statusCode >= 500 || statusCode === 408) + return ResponseCategory.SERVER_ERROR; + if (statusCode === 429) return ResponseCategory.RATE_LIMITED; + if (statusCode === 413) return ResponseCategory.PAYLOAD_TOO_LARGE; + if (statusCode === 401 || statusCode === 403 || statusCode === 404) + return ResponseCategory.FATAL_CONFIG; + return ResponseCategory.CLIENT_ERROR; +}