Merge pull request #40 from nlile/fix/codex-large-jsonl-readline-crash

fix(codex): handle huge session files

Author: yigitkonur

.github/instructions/parsers.instructions.md

@@ -37,8 +37,8 @@ Both must be registered in `src/parsers/registry.ts` with all `ToolAdapter` fiel
 
 ## JSONL Streaming
 
-- Stream JSONL with `readline.createInterface` — never `fs.readFileSync` for session files
-- Use helpers from `src/utils/jsonl.ts` (`readJsonlFile`, `scanJsonlHead`) when applicable
+- Stream JSONL with helpers from `src/utils/jsonl.ts` (`readJsonlFile`, `scanJsonlHead`) when applicable
+- Never use `fs.readFileSync` for session files
 - Keep only the last ~10 messages in `recentMessages` — do not accumulate the entire conversation
 
 ## Tool Summarizer

AGENTS.md

@@ -17,7 +17,7 @@ Agent behavior instructions for `continues` — the cross-tool AI session handof
 - **Logging**: use `logger` from `src/logger.ts` for all diagnostic output. Never use bare `console.log`/`console.warn`/`console.error` in library code. TUI display goes through `@clack/prompts` or `chalk` via the display layer.
 - **Error types**: throw typed errors from `src/errors.ts` on user-facing paths, not bare `new Error()`.
 - **Tool activity**: use `SummaryCollector` from `src/utils/tool-summarizer.ts` in every parser. Do not build `ToolUsageSummary[]` arrays manually.
-- **JSONL**: stream with `readline.createInterface`, never `fs.readFileSync` + `split('\n')`.
+- **JSONL**: use the shared streaming helpers in `src/utils/jsonl.ts`, never `fs.readFileSync` + `split('\n')`.
 - **SQLite** (OpenCode, Crush parsers): use built-in `node:sqlite` — do not add third-party SQLite dependencies.
 - **Biome rules in force**: `noEmptyBlockStatements` (error), `noUnusedImports` (error), `useConst` (error). Empty `catch {}` blocks fail the linter; use `catch (err) { logger.debug(...) }` instead.
 
@@ -45,7 +45,7 @@ All five steps are required. Missing any one is a bug. See `CLAUDE.md` for detai
 
 - **Writing to tool storage directories** — the tool is read-only. Any write to `~/.claude/`, `~/.codex/`, etc. is a severe bug.
 - **`exec()` with string interpolation** — always use `spawn()` with an argument array in `resume.ts`. Session IDs and paths can contain shell metacharacters.
-- **`fs.readFileSync`/`fs.writeFileSync` in parsers** — these block the event loop. Use async fs APIs or `readline`.
+- **`fs.readFileSync`/`fs.writeFileSync` in parsers** — these block the event loop. Use async fs APIs or shared streaming helpers.
 - **Duplicating parser-helpers** — `cleanSummary`, `extractRepoFromCwd`, `homeDir` live in `src/utils/parser-helpers.ts`. Import them; do not reimplement.
 - **Hardcoding tool names** — derive from `TOOL_NAMES` or `SessionSource`. Never write `if (tool === 'claude' || tool === 'codex' || ...)`.
 - **Importing `node:sqlite` outside OpenCode/Crush parsers** — SQLite is only needed for those two tools. Do not spread this dependency.

REVIEW.md

@@ -16,7 +16,7 @@ Review guidelines for the `continues` CLI tool — a read-only session parser an
 - Use `process.exitCode = N` instead of `process.exit(N)`.
 - Biome handles linting and formatting — do not introduce ESLint or Prettier configs.
 - Parser functions must return `Promise<UnifiedSession[]>` and `Promise<SessionContext>` respectively. Both must be registered in `src/parsers/registry.ts`.
-- JSONL parsing must stream with `readline.createInterface` — do not load entire files into memory with `fs.readFileSync`.
+- JSONL parsing must use the shared streaming helpers in `src/utils/jsonl.ts` where possible — do not load entire files into memory with `fs.readFileSync`.
 - Use the `SummaryCollector` class from `src/utils/tool-summarizer.ts` for tool activity summaries. Do not manually build summary arrays.
 - Shared helpers (`cleanSummary`, `extractRepoFromCwd`, `homeDir`) live in `src/utils/parser-helpers.ts`. Do not duplicate these in individual parsers.
 - Error hierarchy: use typed errors from `src/errors.ts` (`ParseError`, `SessionNotFoundError`, `ToolNotAvailableError`, `UnknownSourceError`, `IndexError`, `StorageError`) rather than bare `throw new Error()` for user-facing error paths.

src/__tests__/shared-utils.test.ts

@@ -3,9 +3,9 @@
  * Covers: jsonl, fs-helpers, content, tool-extraction, parser-helpers additions.
  */
 
-import * as fs from 'fs';
-import * as os from 'os';
-import * as path from 'path';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
 import { afterEach, describe, expect, it } from 'vitest';
 import type { ConversationMessage } from '../types/index.js';
 import { classifyToolName } from '../types/tool-names.js';
@@ -66,6 +66,15 @@ describe('readJsonlFile', () => {
     expect(result).toHaveLength(2);
   });
 
+  it('skips oversized lines without buffering them', async () => {
+    const dir = makeTmpDir();
+    const file = path.join(dir, 'test.jsonl');
+    fs.writeFileSync(file, '{"ok":true}\n{"payload":"abcdefghijklmnopqrstuvwxyz"}\n{"ok":false}\n');
+
+    const result = await readJsonlFile<{ ok: boolean }>(file, { maxLineChars: 12 });
+    expect(result).toEqual([{ ok: true }, { ok: false }]);
+  });
+
   it('returns empty array for non-existent file', async () => {
     const result = await readJsonlFile('/tmp/nonexistent-file.jsonl');
     expect(result).toEqual([]);
@@ -108,6 +117,44 @@ describe('scanJsonlHead', () => {
     await scanJsonlHead('/tmp/nonexistent.jsonl', 10, () => 'continue');
     // No error thrown
   });
+
+  it('keeps scanning after skipping an oversized line', async () => {
+    const dir = makeTmpDir();
+    const file = path.join(dir, 'test.jsonl');
+    fs.writeFileSync(file, '{"i":0}\n{"payload":"abcdefghijklmnopqrstuvwxyz"}\n{"i":2}\n');
+
+    const visited: number[] = [];
+    await scanJsonlHead(
+      file,
+      5,
+      (parsed) => {
+        visited.push((parsed as { i: number }).i);
+        return 'continue';
+      },
+      { maxLineChars: 10 },
+    );
+
+    expect(visited).toEqual([0, 2]);
+  });
+
+  it('stops at the configured byte limit without visiting partial lines', async () => {
+    const dir = makeTmpDir();
+    const file = path.join(dir, 'test.jsonl');
+    fs.writeFileSync(file, '{"i":0}\n{"i":1}\n');
+
+    const visited: number[] = [];
+    await scanJsonlHead(
+      file,
+      5,
+      (parsed) => {
+        visited.push((parsed as { i: number }).i);
+        return 'continue';
+      },
+      { maxBytes: 12 },
+    );
+
+    expect(visited).toEqual([0]);
+  });
 });
 
 describe('getFileStats', () => {
@@ -120,6 +167,15 @@ describe('getFileStats', () => {
     expect(stats.lines).toBe(3);
     expect(stats.bytes).toBeGreaterThan(0);
   });
+
+  it('counts a final unterminated line', async () => {
+    const dir = makeTmpDir();
+    const file = path.join(dir, 'test.jsonl');
+    fs.writeFileSync(file, '{"a":1}\n{"a":2}');
+
+    const stats = await getFileStats(file);
+    expect(stats.lines).toBe(2);
+  });
 });
 
 // ── fs-helpers.ts ────────────────────────────────────────────────────────────
@@ -604,7 +660,9 @@ describe('extractAnthropicToolData structured data', () => {
     const messages: AnthropicMessage[] = [
       {
         role: 'assistant',
-        content: [{ type: 'tool_use', id: 'tu1', name: 'Read', input: { file_path: '/src/app.ts', offset: 10, limit: 50 } }],
+        content: [
+          { type: 'tool_use', id: 'tu1', name: 'Read', input: { file_path: '/src/app.ts', offset: 10, limit: 50 } },
+        ],
       },
     ];
 
@@ -685,7 +743,9 @@ describe('extractAnthropicToolData structured data', () => {
       },
       {
         role: 'user',
-        content: [{ type: 'tool_result', tool_use_id: 'tu1', content: 'Found 3 matches\nsrc/a.ts\nsrc/b.ts\nsrc/c.ts' }],
+        content: [
+          { type: 'tool_result', tool_use_id: 'tu1', content: 'Found 3 matches\nsrc/a.ts\nsrc/b.ts\nsrc/c.ts' },
+        ],
       },
     ];
 
@@ -724,7 +784,14 @@ describe('extractAnthropicToolData structured data', () => {
     const messages: AnthropicMessage[] = [
       {
         role: 'assistant',
-        content: [{ type: 'tool_use', id: 'tu1', name: 'mcp__github__list_issues', input: { repo: 'test/repo', state: 'open' } }],
+        content: [
+          {
+            type: 'tool_use',
+            id: 'tu1',
+            name: 'mcp__github__list_issues',
+            input: { repo: 'test/repo', state: 'open' },
+          },
+        ],
       },
       {
         role: 'user',
@@ -750,7 +817,9 @@ describe('extractAnthropicToolData structured data', () => {
       },
       {
         role: 'user',
-        content: [{ type: 'tool_result', tool_use_id: 'tu1', content: 'Permission denied\nexit code 1', is_error: true }],
+        content: [
+          { type: 'tool_result', tool_use_id: 'tu1', content: 'Permission denied\nexit code 1', is_error: true },
+        ],
       },
     ];
     const { summaries } = extractAnthropicToolData(messages);

src/parsers/codex.ts

@@ -30,6 +30,9 @@ const CODEX_HOME_DIR = process.env.CODEX_HOME || path.join(homeDir(), '.codex');
 const CODEX_SESSIONS_DIR = path.join(CODEX_HOME_DIR, 'sessions');
 const CODEX_ARCHIVED_SESSIONS_DIR = path.join(CODEX_HOME_DIR, 'archived_sessions');
 
+const MAX_EXACT_LINE_COUNT_BYTES = 1024 * 1024;
+const MAX_METADATA_SCAN_BYTES = 1024 * 1024;
+
 /**
  * Find all Codex session files recursively
  */
@@ -51,26 +54,34 @@ async function parseSessionInfo(filePath: string): Promise<{
   let meta: CodexSessionMeta | null = null;
   let firstUserMessage = '';
 
-  await scanJsonlHead(filePath, 150, (parsed) => {
-    const msg = parsed as Record<string, unknown>;
+  await scanJsonlHead(
+    filePath,
+    150,
+    (parsed) => {
+      const msg = parsed as Record<string, unknown>;
 
-    if (msg.type === 'session_meta' && !meta) {
-      meta = msg as unknown as CodexSessionMeta;
-    }
+      if (msg.type === 'session_meta' && !meta) {
+        meta = msg as unknown as CodexSessionMeta;
+      }
 
-    if (!firstUserMessage && msg.type === 'event_msg') {
-      const payload = msg.payload as Record<string, unknown> | undefined;
-      if (payload?.type === 'user_message') {
-        firstUserMessage = (payload.message as string) || '';
+      if (!firstUserMessage && msg.type === 'event_msg') {
+        const payload = msg.payload as Record<string, unknown> | undefined;
+        if (payload?.type === 'user_message') {
+          firstUserMessage = (payload.message as string) || '';
+        }
       }
-    }
 
-    if (!firstUserMessage && msg.type === 'message' && (msg as Record<string, unknown>).role === 'user') {
-      firstUserMessage = typeof msg.content === 'string' ? (msg.content as string) : '';
-    }
+      if (!firstUserMessage && msg.type === 'message' && (msg as Record<string, unknown>).role === 'user') {
+        firstUserMessage = typeof msg.content === 'string' ? (msg.content as string) : '';
+      }
 
-    return 'continue';
-  });
+      if (meta && firstUserMessage) {
+        return 'stop';
+      }
+      return 'continue';
+    },
+    { maxBytes: MAX_METADATA_SCAN_BYTES },
+  );
 
   return { meta, firstUserMessage };
 }
@@ -103,8 +114,11 @@ export async function parseCodexSessions(): Promise<UnifiedSession[]> {
       if (!parsed) continue;
 
       const { meta, firstUserMessage } = await parseSessionInfo(filePath);
-      const stats = await getFileStats(filePath);
       const fileStats = fs.statSync(filePath);
+      const stats =
+        fileStats.size > MAX_EXACT_LINE_COUNT_BYTES
+          ? { lines: 0, bytes: fileStats.size }
+          : await getFileStats(filePath);
 
       const cwd = meta?.payload?.cwd || '';
       const gitUrl = meta?.payload?.git?.repository_url;