From 0896c6485830ca5d09fa50a669c0f273c9b3c21a Mon Sep 17 00:00:00 2001 From: MumuTW Date: Fri, 6 Mar 2026 03:57:04 +0000 Subject: [PATCH 1/6] fix(ccusage): stream context token parsing and skip file-history snapshots --- apps/ccusage/src/data-loader.ts | 166 ++++++++++++++++++++------------ 1 file changed, 103 insertions(+), 63 deletions(-) diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts index 8e22aae2..dbc1400f 100644 --- a/apps/ccusage/src/data-loader.ts +++ b/apps/ccusage/src/data-loader.ts @@ -13,7 +13,6 @@ import type { LoadedUsageEntry, SessionBlock } from './_session-blocks.ts'; import type { ActivityDate, Bucket, CostMode, ModelName, SortOrder, Version } from './_types.ts'; import { Buffer } from 'node:buffer'; import { createReadStream, createWriteStream } from 'node:fs'; -import { readFile } from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; import { createInterface } from 'node:readline'; @@ -1259,80 +1258,100 @@ export async function calculateContextTokens( percentage: number; contextLimit: number; } | null> { - let content: string; + let latestUsage: { + input_tokens: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; + } | null = null; + let firstNonEmptyLineSeen = false; + try { - content = await readFile(transcriptPath, 'utf-8'); - } catch (error: unknown) { - logger.debug(`Failed to read transcript file: ${String(error)}`); - return null; - } + const stream = createReadStream(transcriptPath, { encoding: 'utf-8' }); + const reader = createInterface({ + input: stream, + crlfDelay: Number.POSITIVE_INFINITY, + }); - const lines = content.split('\n').reverse(); // Iterate from last line to first line + for await (const line of reader) { + const trimmedLine = line.trim(); + if (trimmedLine === '') { + continue; + } - for (const line of lines) { - const trimmedLine = line.trim(); - if (trimmedLine === '') { - continue; - } + if (!firstNonEmptyLineSeen) { + firstNonEmptyLineSeen = true; - try { - const parsed = JSON.parse(trimmedLine) as unknown; - const result = v.safeParse(transcriptMessageSchema, parsed); - if (!result.success) { - continue; // Skip malformed JSON lines - } - const obj = result.output; - - // Check if this line contains the required token usage fields - if ( - obj.type === 'assistant' && - obj.message != null && - obj.message.usage != null && - obj.message.usage.input_tokens != null - ) { - const usage = obj.message.usage; - const inputTokens = - usage.input_tokens! + - (usage.cache_creation_input_tokens ?? 0) + - (usage.cache_read_input_tokens ?? 0); - - // Get context limit from PricingFetcher - let contextLimit = 200_000; // Fallback for when modelId is not provided - if (modelId != null && modelId !== '') { - using fetcher = new PricingFetcher(offline); - const contextLimitResult = await fetcher.getModelContextLimit(modelId); - if (Result.isSuccess(contextLimitResult) && contextLimitResult.value != null) { - contextLimit = contextLimitResult.value; - } else if (Result.isSuccess(contextLimitResult)) { - // Context limit not available for this model in LiteLLM - logger.debug(`No context limit data available for model ${modelId} in LiteLLM`); - } else { - // Error occurred - logger.debug( - `Failed to get context limit for model ${modelId}: ${contextLimitResult.error.message}`, - ); + try { + const parsed = JSON.parse(trimmedLine) as { type?: string }; + if (parsed.type === 'file-history-snapshot') { + logger.debug('Skipping file-history-snapshot transcript file for context tokens'); + return null; } + } catch { + // Continue to normal parsing for malformed first lines } + } - const percentage = Math.min( - 100, - Math.max(0, Math.round((inputTokens / contextLimit) * 100)), - ); + try { + const parsed = JSON.parse(trimmedLine) as unknown; + const result = v.safeParse(transcriptMessageSchema, parsed); + if (!result.success) { + continue; + } - return { - inputTokens, - percentage, - contextLimit, - }; + const obj = result.output; + if ( + obj.type === 'assistant' && + obj.message != null && + obj.message.usage != null && + obj.message.usage.input_tokens != null + ) { + latestUsage = obj.message.usage; + } + } catch { + // Skip malformed JSON lines } - } catch { - continue; // Skip malformed JSON lines + } + } catch (error: unknown) { + logger.debug(`Failed to read transcript file: ${String(error)}`); + return null; + } + + if (latestUsage == null) { + logger.debug('No usage information found in transcript'); + return null; + } + + const inputTokens = + latestUsage.input_tokens + + (latestUsage.cache_creation_input_tokens ?? 0) + + (latestUsage.cache_read_input_tokens ?? 0); + + // Get context limit from PricingFetcher + let contextLimit = 200_000; // Fallback for when modelId is not provided + if (modelId != null && modelId !== '') { + using fetcher = new PricingFetcher(offline); + const contextLimitResult = await fetcher.getModelContextLimit(modelId); + if (Result.isSuccess(contextLimitResult) && contextLimitResult.value != null) { + contextLimit = contextLimitResult.value; + } else if (Result.isSuccess(contextLimitResult)) { + // Context limit not available for this model in LiteLLM + logger.debug(`No context limit data available for model ${modelId} in LiteLLM`); + } else { + // Error occurred + logger.debug( + `Failed to get context limit for model ${modelId}: ${contextLimitResult.error.message}`, + ); } } - // No valid usage information found - logger.debug('No usage information found in transcript'); - return null; + const percentage = Math.min(100, Math.max(0, Math.round((inputTokens / contextLimit) * 100))); + + return { + inputTokens, + percentage, + contextLimit, + }; } /** @@ -4749,5 +4768,26 @@ if (import.meta.vitest != null) { expect(res).not.toBeNull(); expect(res?.percentage).toBe(100); // Should be clamped to 100 }); + + it('skips file-history-snapshot transcripts early', async () => { + await using fixture = await createFixture({ + 'transcript.jsonl': [ + JSON.stringify({ + type: 'file-history-snapshot', + entries: Array.from({ length: 10_000 }, (_, i) => ({ + path: `src/file-${i}.ts`, + hash: `hash-${i}`, + })), + }), + JSON.stringify({ + type: 'assistant', + message: { usage: { input_tokens: 1234 } }, + }), + ].join('\n'), + }); + + const res = await calculateContextTokens(fixture.getPath('transcript.jsonl')); + expect(res).toBeNull(); + }); }); } From 9995939ed0548e36eab0a507b73633917d998d09 Mon Sep 17 00:00:00 2001 From: MumuTW Date: Fri, 6 Mar 2026 05:09:55 +0000 Subject: [PATCH 2/6] fix(opencode): silence logger in json output mode --- apps/opencode/src/commands/daily.ts | 3 +++ apps/opencode/src/commands/monthly.ts | 3 +++ apps/opencode/src/commands/session.ts | 3 +++ apps/opencode/src/commands/weekly.ts | 3 +++ 4 files changed, 12 insertions(+) diff --git a/apps/opencode/src/commands/daily.ts b/apps/opencode/src/commands/daily.ts index 1ad9b1a8..6a0e05d3 100644 --- a/apps/opencode/src/commands/daily.ts +++ b/apps/opencode/src/commands/daily.ts @@ -32,6 +32,9 @@ export const dailyCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const entries = await loadOpenCodeMessages(); diff --git a/apps/opencode/src/commands/monthly.ts b/apps/opencode/src/commands/monthly.ts index 453795c5..cabfc7ad 100644 --- a/apps/opencode/src/commands/monthly.ts +++ b/apps/opencode/src/commands/monthly.ts @@ -32,6 +32,9 @@ export const monthlyCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const entries = await loadOpenCodeMessages(); diff --git a/apps/opencode/src/commands/session.ts b/apps/opencode/src/commands/session.ts index c36467c0..c37e1b15 100644 --- a/apps/opencode/src/commands/session.ts +++ b/apps/opencode/src/commands/session.ts @@ -32,6 +32,9 @@ export const sessionCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const [entries, sessionMetadataMap] = await Promise.all([ loadOpenCodeMessages(), diff --git a/apps/opencode/src/commands/weekly.ts b/apps/opencode/src/commands/weekly.ts index 011e8204..37e3ce28 100644 --- a/apps/opencode/src/commands/weekly.ts +++ b/apps/opencode/src/commands/weekly.ts @@ -57,6 +57,9 @@ export const weeklyCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const entries = await loadOpenCodeMessages(); From b9fd7cbb045c0232939fbeb6f6147d8aa7ae495c Mon Sep 17 00:00:00 2001 From: MumuTW Date: Fri, 6 Mar 2026 11:17:32 +0000 Subject: [PATCH 3/6] fix(ccusage): narrow usage type to satisfy strict input_tokens requirement The latestUsage variable requires input_tokens as a non-optional number, but obj.message.usage has it as optional. Explicitly construct the object after the null check so TypeScript can see the narrowed type. Co-Authored-By: Claude Opus 4.6 --- apps/ccusage/src/data-loader.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts index dbc1400f..3fb93623 100644 --- a/apps/ccusage/src/data-loader.ts +++ b/apps/ccusage/src/data-loader.ts @@ -1306,7 +1306,11 @@ export async function calculateContextTokens( obj.message.usage != null && obj.message.usage.input_tokens != null ) { - latestUsage = obj.message.usage; + latestUsage = { + input_tokens: obj.message.usage.input_tokens, + cache_creation_input_tokens: obj.message.usage.cache_creation_input_tokens, + cache_read_input_tokens: obj.message.usage.cache_read_input_tokens, + }; } } catch { // Skip malformed JSON lines From a3bc6cbfdf5cf2298f1c1ecaf2e73cc878c1207a Mon Sep 17 00:00:00 2001 From: MumuTW Date: Fri, 6 Mar 2026 11:34:36 +0000 Subject: [PATCH 4/6] fix(ccusage): use bounded prefix read for file-history-snapshot detection Read only the first 4 KiB of the file to detect file-history-snapshot type instead of using readline, which buffers the entire first line and crashes on huge single-line records (e.g. 734 MB). Co-Authored-By: Claude Opus 4.6 --- apps/ccusage/src/data-loader.ts | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts index 3fb93623..73391ff6 100644 --- a/apps/ccusage/src/data-loader.ts +++ b/apps/ccusage/src/data-loader.ts @@ -12,7 +12,7 @@ import type { WeekDay } from './_consts.ts'; import type { LoadedUsageEntry, SessionBlock } from './_session-blocks.ts'; import type { ActivityDate, Bucket, CostMode, ModelName, SortOrder, Version } from './_types.ts'; import { Buffer } from 'node:buffer'; -import { createReadStream, createWriteStream } from 'node:fs'; +import { closeSync, createReadStream, createWriteStream, openSync, readSync } from 'node:fs'; import path from 'node:path'; import process from 'node:process'; import { createInterface } from 'node:readline'; @@ -1263,9 +1263,27 @@ export async function calculateContextTokens( cache_creation_input_tokens?: number; cache_read_input_tokens?: number; } | null = null; - let firstNonEmptyLineSeen = false; - try { + // Fast-path: read a small prefix to detect file-history-snapshot without + // buffering a potentially huge first line via readline (see #873). + const PREFIX_SIZE = 4096; + const prefixBuf = Buffer.alloc(PREFIX_SIZE); + const fd = openSync(transcriptPath, 'r'); + let bytesRead: number; + try { + bytesRead = readSync(fd, prefixBuf, 0, PREFIX_SIZE, 0); + } finally { + closeSync(fd); + } + if (bytesRead > 0) { + const prefix = prefixBuf.subarray(0, bytesRead).toString('utf-8').trimStart(); + const typeMatch = prefix.match(/^\s*\{\s*"type"\s*:\s*"([^"]+)"/); + if (typeMatch != null && typeMatch[1] === 'file-history-snapshot') { + logger.debug('Skipping file-history-snapshot transcript file for context tokens'); + return null; + } + } + const stream = createReadStream(transcriptPath, { encoding: 'utf-8' }); const reader = createInterface({ input: stream, @@ -1278,20 +1296,6 @@ export async function calculateContextTokens( continue; } - if (!firstNonEmptyLineSeen) { - firstNonEmptyLineSeen = true; - - try { - const parsed = JSON.parse(trimmedLine) as { type?: string }; - if (parsed.type === 'file-history-snapshot') { - logger.debug('Skipping file-history-snapshot transcript file for context tokens'); - return null; - } - } catch { - // Continue to normal parsing for malformed first lines - } - } - try { const parsed = JSON.parse(trimmedLine) as unknown; const result = v.safeParse(transcriptMessageSchema, parsed); From e16a183320245e881fa978034a687e2ac1f14900 Mon Sep 17 00:00:00 2001 From: MumuTW Date: Sat, 7 Mar 2026 21:25:48 +0000 Subject: [PATCH 5/6] fix: detect file-history-snapshot regardless of key order The fast-path regex previously required "type" to be the first key in the JSON object. If a serializer placed another field first (e.g. "version"), the snapshot would slip through to readline and re-trigger the large-line crash. Now we search for "type":"file-history-snapshot" anywhere in the first line of the 4 KiB prefix, matching any key order. --- apps/ccusage/src/data-loader.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts index 73391ff6..49439e5f 100644 --- a/apps/ccusage/src/data-loader.ts +++ b/apps/ccusage/src/data-loader.ts @@ -1276,9 +1276,11 @@ export async function calculateContextTokens( closeSync(fd); } if (bytesRead > 0) { - const prefix = prefixBuf.subarray(0, bytesRead).toString('utf-8').trimStart(); - const typeMatch = prefix.match(/^\s*\{\s*"type"\s*:\s*"([^"]+)"/); - if (typeMatch != null && typeMatch[1] === 'file-history-snapshot') { + const prefix = prefixBuf.subarray(0, bytesRead).toString('utf-8'); + // Extract the first line (up to the first newline) and look for the + // type field anywhere in it — not just as the first key. + const firstLine = prefix.split('\n', 1)[0] ?? ''; + if (/^\s*\{/.test(firstLine) && /"type"\s*:\s*"file-history-snapshot"/.test(firstLine)) { logger.debug('Skipping file-history-snapshot transcript file for context tokens'); return null; } @@ -4797,5 +4799,24 @@ if (import.meta.vitest != null) { const res = await calculateContextTokens(fixture.getPath('transcript.jsonl')); expect(res).toBeNull(); }); + + it('skips file-history-snapshot even when type is not the first key', async () => { + await using fixture = await createFixture({ + 'transcript.jsonl': [ + JSON.stringify({ + version: 1, + type: 'file-history-snapshot', + entries: [{ path: 'a.ts', hash: 'h' }], + }), + JSON.stringify({ + type: 'assistant', + message: { usage: { input_tokens: 500 } }, + }), + ].join('\n'), + }); + + const res = await calculateContextTokens(fixture.getPath('transcript.jsonl')); + expect(res).toBeNull(); + }); }); } From 95ab9a2fb88e7704a82bd15dac106ecc8ab879ce Mon Sep 17 00:00:00 2001 From: MumuTW Date: Sun, 8 Mar 2026 01:54:23 +0000 Subject: [PATCH 6/6] fix: probe first non-empty line in snapshot fast-path Files starting with blank lines before the snapshot record would miss the fast-path check and fall back to readline, re-opening the large-line buffering issue. Use .find() to skip empty leading lines. --- apps/ccusage/src/data-loader.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts index 49439e5f..597e7727 100644 --- a/apps/ccusage/src/data-loader.ts +++ b/apps/ccusage/src/data-loader.ts @@ -1279,7 +1279,8 @@ export async function calculateContextTokens( const prefix = prefixBuf.subarray(0, bytesRead).toString('utf-8'); // Extract the first line (up to the first newline) and look for the // type field anywhere in it — not just as the first key. - const firstLine = prefix.split('\n', 1)[0] ?? ''; + // Find the first non-empty line so leading blank lines don't hide the snapshot record. + const firstLine = prefix.split('\n').find((l) => l.trim() !== '') ?? ''; if (/^\s*\{/.test(firstLine) && /"type"\s*:\s*"file-history-snapshot"/.test(firstLine)) { logger.debug('Skipping file-history-snapshot transcript file for context tokens'); return null;