From 19d7000f89be56a16d89b385729d1b2c97e66a4e Mon Sep 17 00:00:00 2001 From: GriggsRice Date: Sat, 11 Jul 2026 00:50:40 -0500 Subject: [PATCH] fix(browseros-agent): strip leaked model scaffold tokens from chat output Some local models (observed via LM Studio) embed their own thinking-scaffold markers, e.g. "<|channel|>" or the malformed "", directly in regular text/reasoning output instead of a structured field. Nothing sanitized this before rendering, so raw protocol tokens showed up verbatim in the chat UI. Add stripScaffoldTokens(), a regex-based sanitizer matching only bracket+pipe token shapes ("<|word|>", "<|word>", ""), so real HTML-like text and markdown tables are untouched. Applied to both text and reasoning segments in getMessageSegments; segments that become empty after stripping are dropped entirely. Co-Authored-By: Claude Sonnet 5 --- .../index/getMessageSegments.test.ts | 93 +++++++++++++++++++ .../sidepanel/index/getMessageSegments.ts | 72 +++++++++++--- 2 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.test.ts diff --git a/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.test.ts b/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.test.ts new file mode 100644 index 0000000000..bf7cf80322 --- /dev/null +++ b/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'bun:test' +import type { UIMessage } from 'ai' +import { getMessageSegments, stripScaffoldTokens } from './getMessageSegments' + +describe('stripScaffoldTokens', () => { + it('strips the malformed delimiter pair observed from LM Studio (missing pipes), leaving the embedded channel-name word', () => { + expect(stripScaffoldTokens('<|channel>thought\n')).toBe('thought') + }) + + it('strips a token-only string down to empty', () => { + expect(stripScaffoldTokens('<|end|>')).toBe('') + expect(stripScaffoldTokens('<|channel|>')).toBe('') + }) + + it('strips well-formed Harmony-style tokens', () => { + expect( + stripScaffoldTokens('<|channel|>final<|message|>The answer is 42<|end|>'), + ).toBe('finalThe answer is 42') + }) + + it('leaves real HTML-like text untouched', () => { + expect(stripScaffoldTokens('Use a
tag here')).toBe( + 'Use a
tag here', + ) + }) + + it('leaves markdown tables untouched', () => { + const table = '| a | b |\n| --- | --- |\n| 1 | 2 |' + expect(stripScaffoldTokens(table)).toBe(table) + }) + + it('collapses excess blank lines left behind after stripping', () => { + expect(stripScaffoldTokens('before\n<|end|>\n\n\n\nafter')).toBe( + 'before\n\nafter', + ) + }) + + it('trims surrounding whitespace', () => { + expect(stripScaffoldTokens(' <|channel|>thought ')).toBe('thought') + }) +}) + +function buildMessage(parts: UIMessage['parts']): UIMessage { + return { id: 'msg-1', role: 'assistant', parts } +} + +describe('getMessageSegments scaffold-token handling', () => { + it('drops a text part that is entirely leaked scaffold tokens', () => { + const message = buildMessage([{ type: 'text', text: '<|end|>' }]) + expect(getMessageSegments(message, false, false)).toEqual([]) + }) + + it('keeps a bare channel-name word left behind when no real content followed', () => { + const message = buildMessage([ + { type: 'text', text: '<|channel>thought\n' }, + ]) + expect(getMessageSegments(message, false, false)).toEqual([ + { type: 'text', key: 'msg-1-text-0', text: 'thought' }, + ]) + }) + + it('keeps real text content once scaffold tokens are stripped', () => { + const message = buildMessage([ + { type: 'text', text: '<|channel|>final<|message|>Hello there<|end|>' }, + ]) + const segments = getMessageSegments(message, false, false) + expect(segments).toEqual([ + { type: 'text', key: 'msg-1-text-0', text: 'finalHello there' }, + ]) + }) + + it('does not disrupt a real tool batch surrounding a leaked segment', () => { + const message = buildMessage([ + { + type: 'tool-search', + toolCallId: 'call-1', + state: 'output-available', + input: {}, + output: {}, + }, + { type: 'text', text: '<|end|>' }, + { + type: 'tool-filesystem_read', + toolCallId: 'call-2', + state: 'input-available', + input: {}, + output: undefined, + }, + ]) + const segments = getMessageSegments(message, true, true) + expect(segments.map((s) => s.type)).toEqual(['tool-batch', 'tool-batch']) + }) +}) diff --git a/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.ts b/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.ts index 5d26e611fe..f8eebaabdf 100644 --- a/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.ts +++ b/packages/browseros-agent/apps/app/screens/sidepanel/index/getMessageSegments.ts @@ -36,6 +36,51 @@ export type MessageSegment = const NUDGE_TOOLS = new Set(['suggest_schedule', 'suggest_app_connection']) +// Some local models (observed via LM Studio) leak raw Harmony-style channel +// scaffold tokens like "<|channel|>" or the malformed "" into +// plain text/reasoning output instead of confining them to a structured +// field. Matches only bracket+pipe token shapes ("<|word|>", "<|word>", +// "") so real HTML-like text (e.g. "
") and markdown tables +// ("| a | b |") are never touched. +const SCAFFOLD_TOKEN_PATTERN = /<\|[a-zA-Z_]+\|?>|<[a-zA-Z_]+\|>/g + +export function stripScaffoldTokens(text: string): string { + return text + .replace(SCAFFOLD_TOKEN_PATTERN, '') + .replace(/\n{3,}/g, '\n\n') + .trim() +} + +function pushTextSegment( + segments: MessageSegment[], + messageId: string, + index: number, + rawText: string, +): number { + const text = stripScaffoldTokens(rawText) + if (!text) return index + segments.push({ type: 'text', key: `${messageId}-text-${index}`, text }) + return index + 1 +} + +function pushReasoningSegment( + segments: MessageSegment[], + messageId: string, + index: number, + rawText: string, + isPartStreaming: boolean, +): number { + const text = stripScaffoldTokens(rawText) + if (!text) return index + segments.push({ + type: 'reasoning', + key: `${messageId}-reasoning-${index}`, + text, + isStreaming: isPartStreaming, + }) + return index + 1 +} + function parseNudgeOutput(output: unknown): NudgeData | null { try { // output is { content: [{ type: "text", text: "JSON..." }], isError: false } @@ -87,22 +132,21 @@ export const getMessageSegments = ( if (part.type === 'text') { flushToolBatch() - segments.push({ - type: 'text', - key: `${message.id}-text-${textSegmentCount}`, - text: part.text, - }) - textSegmentCount++ + textSegmentCount = pushTextSegment( + segments, + message.id, + textSegmentCount, + part.text, + ) } else if (part.type === 'reasoning') { flushToolBatch() - segments.push({ - type: 'reasoning', - key: `${message.id}-reasoning-${reasoningSegmentCount}`, - text: part.text, - isStreaming: - isStreaming && i === message.parts.length - 1 && isLastMessage, - }) - reasoningSegmentCount++ + reasoningSegmentCount = pushReasoningSegment( + segments, + message.id, + reasoningSegmentCount, + part.text, + isStreaming && i === message.parts.length - 1 && isLastMessage, + ) } else if (part.type?.startsWith('tool-') || part.type === 'dynamic-tool') { const toolPart = part as { toolCallId: string