diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts index 8e22aae2..597e7727 100644 --- a/apps/ccusage/src/data-loader.ts +++ b/apps/ccusage/src/data-loader.ts @@ -12,8 +12,7 @@ import type { WeekDay } from './_consts.ts'; import type { LoadedUsageEntry, SessionBlock } from './_session-blocks.ts'; import type { ActivityDate, Bucket, CostMode, ModelName, SortOrder, Version } from './_types.ts'; import { Buffer } from 'node:buffer'; -import { createReadStream, createWriteStream } from 'node:fs'; -import { readFile } from 'node:fs/promises'; +import { closeSync, createReadStream, createWriteStream, openSync, readSync } from 'node:fs'; import path from 'node:path'; import process from 'node:process'; import { createInterface } from 'node:readline'; @@ -1259,80 +1258,111 @@ export async function calculateContextTokens( percentage: number; contextLimit: number; } | null> { - let content: string; + let latestUsage: { + input_tokens: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; + } | null = null; try { - content = await readFile(transcriptPath, 'utf-8'); - } catch (error: unknown) { - logger.debug(`Failed to read transcript file: ${String(error)}`); - return null; - } - - const lines = content.split('\n').reverse(); // Iterate from last line to first line - - for (const line of lines) { - const trimmedLine = line.trim(); - if (trimmedLine === '') { - continue; + // Fast-path: read a small prefix to detect file-history-snapshot without + // buffering a potentially huge first line via readline (see #873). + const PREFIX_SIZE = 4096; + const prefixBuf = Buffer.alloc(PREFIX_SIZE); + const fd = openSync(transcriptPath, 'r'); + let bytesRead: number; + try { + bytesRead = readSync(fd, prefixBuf, 0, PREFIX_SIZE, 0); + } finally { + closeSync(fd); + } + if (bytesRead > 0) { + const prefix = prefixBuf.subarray(0, bytesRead).toString('utf-8'); + // Extract the first line (up to the first newline) and look for the + // type field anywhere in it — not just as the first key. + // Find the first non-empty line so leading blank lines don't hide the snapshot record. + const firstLine = prefix.split('\n').find((l) => l.trim() !== '') ?? ''; + if (/^\s*\{/.test(firstLine) && /"type"\s*:\s*"file-history-snapshot"/.test(firstLine)) { + logger.debug('Skipping file-history-snapshot transcript file for context tokens'); + return null; + } } - try { - const parsed = JSON.parse(trimmedLine) as unknown; - const result = v.safeParse(transcriptMessageSchema, parsed); - if (!result.success) { - continue; // Skip malformed JSON lines + const stream = createReadStream(transcriptPath, { encoding: 'utf-8' }); + const reader = createInterface({ + input: stream, + crlfDelay: Number.POSITIVE_INFINITY, + }); + + for await (const line of reader) { + const trimmedLine = line.trim(); + if (trimmedLine === '') { + continue; } - const obj = result.output; - - // Check if this line contains the required token usage fields - if ( - obj.type === 'assistant' && - obj.message != null && - obj.message.usage != null && - obj.message.usage.input_tokens != null - ) { - const usage = obj.message.usage; - const inputTokens = - usage.input_tokens! + - (usage.cache_creation_input_tokens ?? 0) + - (usage.cache_read_input_tokens ?? 0); - - // Get context limit from PricingFetcher - let contextLimit = 200_000; // Fallback for when modelId is not provided - if (modelId != null && modelId !== '') { - using fetcher = new PricingFetcher(offline); - const contextLimitResult = await fetcher.getModelContextLimit(modelId); - if (Result.isSuccess(contextLimitResult) && contextLimitResult.value != null) { - contextLimit = contextLimitResult.value; - } else if (Result.isSuccess(contextLimitResult)) { - // Context limit not available for this model in LiteLLM - logger.debug(`No context limit data available for model ${modelId} in LiteLLM`); - } else { - // Error occurred - logger.debug( - `Failed to get context limit for model ${modelId}: ${contextLimitResult.error.message}`, - ); - } - } - const percentage = Math.min( - 100, - Math.max(0, Math.round((inputTokens / contextLimit) * 100)), - ); + try { + const parsed = JSON.parse(trimmedLine) as unknown; + const result = v.safeParse(transcriptMessageSchema, parsed); + if (!result.success) { + continue; + } - return { - inputTokens, - percentage, - contextLimit, - }; + const obj = result.output; + if ( + obj.type === 'assistant' && + obj.message != null && + obj.message.usage != null && + obj.message.usage.input_tokens != null + ) { + latestUsage = { + input_tokens: obj.message.usage.input_tokens, + cache_creation_input_tokens: obj.message.usage.cache_creation_input_tokens, + cache_read_input_tokens: obj.message.usage.cache_read_input_tokens, + }; + } + } catch { + // Skip malformed JSON lines } - } catch { - continue; // Skip malformed JSON lines + } + } catch (error: unknown) { + logger.debug(`Failed to read transcript file: ${String(error)}`); + return null; + } + + if (latestUsage == null) { + logger.debug('No usage information found in transcript'); + return null; + } + + const inputTokens = + latestUsage.input_tokens + + (latestUsage.cache_creation_input_tokens ?? 0) + + (latestUsage.cache_read_input_tokens ?? 0); + + // Get context limit from PricingFetcher + let contextLimit = 200_000; // Fallback for when modelId is not provided + if (modelId != null && modelId !== '') { + using fetcher = new PricingFetcher(offline); + const contextLimitResult = await fetcher.getModelContextLimit(modelId); + if (Result.isSuccess(contextLimitResult) && contextLimitResult.value != null) { + contextLimit = contextLimitResult.value; + } else if (Result.isSuccess(contextLimitResult)) { + // Context limit not available for this model in LiteLLM + logger.debug(`No context limit data available for model ${modelId} in LiteLLM`); + } else { + // Error occurred + logger.debug( + `Failed to get context limit for model ${modelId}: ${contextLimitResult.error.message}`, + ); } } - // No valid usage information found - logger.debug('No usage information found in transcript'); - return null; + const percentage = Math.min(100, Math.max(0, Math.round((inputTokens / contextLimit) * 100))); + + return { + inputTokens, + percentage, + contextLimit, + }; } /** @@ -4749,5 +4779,45 @@ if (import.meta.vitest != null) { expect(res).not.toBeNull(); expect(res?.percentage).toBe(100); // Should be clamped to 100 }); + + it('skips file-history-snapshot transcripts early', async () => { + await using fixture = await createFixture({ + 'transcript.jsonl': [ + JSON.stringify({ + type: 'file-history-snapshot', + entries: Array.from({ length: 10_000 }, (_, i) => ({ + path: `src/file-${i}.ts`, + hash: `hash-${i}`, + })), + }), + JSON.stringify({ + type: 'assistant', + message: { usage: { input_tokens: 1234 } }, + }), + ].join('\n'), + }); + + const res = await calculateContextTokens(fixture.getPath('transcript.jsonl')); + expect(res).toBeNull(); + }); + + it('skips file-history-snapshot even when type is not the first key', async () => { + await using fixture = await createFixture({ + 'transcript.jsonl': [ + JSON.stringify({ + version: 1, + type: 'file-history-snapshot', + entries: [{ path: 'a.ts', hash: 'h' }], + }), + JSON.stringify({ + type: 'assistant', + message: { usage: { input_tokens: 500 } }, + }), + ].join('\n'), + }); + + const res = await calculateContextTokens(fixture.getPath('transcript.jsonl')); + expect(res).toBeNull(); + }); }); } diff --git a/apps/opencode/src/commands/daily.ts b/apps/opencode/src/commands/daily.ts index 1ad9b1a8..6a0e05d3 100644 --- a/apps/opencode/src/commands/daily.ts +++ b/apps/opencode/src/commands/daily.ts @@ -32,6 +32,9 @@ export const dailyCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const entries = await loadOpenCodeMessages(); diff --git a/apps/opencode/src/commands/monthly.ts b/apps/opencode/src/commands/monthly.ts index 453795c5..cabfc7ad 100644 --- a/apps/opencode/src/commands/monthly.ts +++ b/apps/opencode/src/commands/monthly.ts @@ -32,6 +32,9 @@ export const monthlyCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const entries = await loadOpenCodeMessages(); diff --git a/apps/opencode/src/commands/session.ts b/apps/opencode/src/commands/session.ts index c36467c0..c37e1b15 100644 --- a/apps/opencode/src/commands/session.ts +++ b/apps/opencode/src/commands/session.ts @@ -32,6 +32,9 @@ export const sessionCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const [entries, sessionMetadataMap] = await Promise.all([ loadOpenCodeMessages(), diff --git a/apps/opencode/src/commands/weekly.ts b/apps/opencode/src/commands/weekly.ts index 011e8204..37e3ce28 100644 --- a/apps/opencode/src/commands/weekly.ts +++ b/apps/opencode/src/commands/weekly.ts @@ -57,6 +57,9 @@ export const weeklyCommand = define({ }, async run(ctx) { const jsonOutput = Boolean(ctx.values.json); + if (jsonOutput) { + logger.level = 0; + } const entries = await loadOpenCodeMessages();