From acf5f93e6d8337b1ea3e3ba683a9b9a9faf68c11 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Sun, 7 Apr 2024 03:21:26 +0300 Subject: [PATCH 1/6] fix: remove {{post}} from when continuing generation --- common/prompt.ts | 6 +----- common/template-parser.ts | 12 ++++++++++++ srv/api/chat/message.ts | 6 +++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/common/prompt.ts b/common/prompt.ts index 63127473b..d2c9f104a 100644 --- a/common/prompt.ts +++ b/common/prompt.ts @@ -422,10 +422,6 @@ export async function buildPromptParts( const post = createPostPrompt(opts) - if (opts.continue) { - post.unshift(`${char.name}: ${opts.continue}`) - } - const linesForMemory = [...lines].reverse() const books: AppSchema.MemoryBook[] = [] if (replyAs.characterBook) books.push(replyAs.characterBook) @@ -514,7 +510,7 @@ function createPostPrompt( if (opts.kind === 'chat-query') { post.push(`Query Response:`) - } else { + } else if (opts.kind !== 'continue') { post.push(`${opts.replyAs.name}:`) } diff --git a/common/template-parser.ts b/common/template-parser.ts index 07f03f914..f8e0a423d 100644 --- a/common/template-parser.ts +++ b/common/template-parser.ts @@ -139,6 +139,18 @@ export async function parseTemplate( } const ast = parser.parse(template, {}) as PNode[] + + // hack: when we continue, remove the post along with the last newline from tree + if (opts.continue && ast.length > 1) { + const last = ast[ast.length - 1] + if (typeof last !== 'string' && last.kind === 'placeholder' && last.value === 'post') { + ast.pop() + } + if (ast[ast.length - 1] === '\n') { + ast.pop() + } + } + readInserts(opts, ast) let output = render(template, opts, ast) let unusedTokens = 0 diff --git a/srv/api/chat/message.ts b/srv/api/chat/message.ts index 120bdac4d..642f4397f 100644 --- a/srv/api/chat/message.ts +++ b/srv/api/chat/message.ts @@ -271,7 +271,7 @@ export const generateMessageV2 = handle(async (req, res) => { } if ('partial' in gen) { - const prefix = body.kind === 'continue' ? `${body.continuing.msg} ` : '' + const prefix = body.kind === 'continue' ? `${body.continuing.msg}` : '' sendMany(members, { type: 'message-partial', kind: body.kind, @@ -331,7 +331,7 @@ export const generateMessageV2 = handle(async (req, res) => { return } - const responseText = body.kind === 'continue' ? `${body.continuing.msg} ${generated}` : generated + const responseText = body.kind === 'continue' ? `${body.continuing.msg}${generated}` : generated const actions: AppSchema.ChatAction[] = [] switch (body.kind) { @@ -568,7 +568,7 @@ async function handleGuestGenerate(body: GenRequest, req: AppRequest, res: Respo if (error) return - const responseText = body.kind === 'continue' ? `${body.continuing.msg} ${generated}` : generated + const responseText = body.kind === 'continue' ? `${body.continuing.msg}${generated}` : generated const characterId = body.kind === 'self' ? undefined : body.replyAs?._id || body.char?._id const senderId = body.kind === 'self' ? 'anon' : undefined From 7d4006c949c3fe6ff0f32064af70cacd79dc0e02 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Sun, 7 Apr 2024 04:19:13 +0300 Subject: [PATCH 2/6] feat: better concat for generation that was continued Note: this approach doesn't account for the event when continuation starts mid-word (white space will be inserted anyway). However, statistically chance of this happening is low. --- common/util.ts | 9 +++++++++ srv/api/chat/message.ts | 10 +++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/common/util.ts b/common/util.ts index 521b44144..83e9ae238 100644 --- a/common/util.ts +++ b/common/util.ts @@ -175,6 +175,15 @@ export function trimSentence(text: string) { return index === -1 ? text.trimEnd() : text.slice(0, index + 1).trimEnd() } +export function concatenateSentence(text: string, next: string) { + if (!text || !next) return text + next + if (text.endsWith('\n') || next.startsWith('\n')) { + return `${text.trimEnd()}\n${next.trimStart()}` + } + + return `${text.trimEnd()} ${next.trimStart()}` +} + export function slugify(str: string) { return str .toLowerCase() diff --git a/srv/api/chat/message.ts b/srv/api/chat/message.ts index 642f4397f..d812f1590 100644 --- a/srv/api/chat/message.ts +++ b/srv/api/chat/message.ts @@ -9,6 +9,7 @@ import { v4 } from 'uuid' import { Response } from 'express' import { publishMany } from '../ws/handle' import { getScenarioEventType } from '/common/scenario' +import { concatenateSentence } from '/common/util' type GenRequest = UnwrapBody @@ -271,7 +272,7 @@ export const generateMessageV2 = handle(async (req, res) => { } if ('partial' in gen) { - const prefix = body.kind === 'continue' ? `${body.continuing.msg}` : '' + const prefix = body.kind === 'continue' ? `${body.continuing.msg} ` : '' sendMany(members, { type: 'message-partial', kind: body.kind, @@ -331,7 +332,9 @@ export const generateMessageV2 = handle(async (req, res) => { return } - const responseText = body.kind === 'continue' ? `${body.continuing.msg}${generated}` : generated + const responseText = + body.kind === 'continue' ? concatenateSentence(body.continuing.msg, generated) : generated + const actions: AppSchema.ChatAction[] = [] switch (body.kind) { @@ -568,7 +571,8 @@ async function handleGuestGenerate(body: GenRequest, req: AppRequest, res: Respo if (error) return - const responseText = body.kind === 'continue' ? `${body.continuing.msg}${generated}` : generated + const responseText = + body.kind === 'continue' ? concatenateSentence(body.continuing.msg, generated) : generated const characterId = body.kind === 'self' ? undefined : body.replyAs?._id || body.char?._id const senderId = body.kind === 'self' ? 'anon' : undefined From 3b05e3341618fb14c5c79a14ba146ddbb28d63fb Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Sun, 7 Apr 2024 18:25:07 +0300 Subject: [PATCH 3/6] test: update snapshot --- tests/__snapshots__/prompt.spec.js.snap | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/__snapshots__/prompt.spec.js.snap b/tests/__snapshots__/prompt.spec.js.snap index f05e4c982..8ccc1ccb6 100644 --- a/tests/__snapshots__/prompt.spec.js.snap +++ b/tests/__snapshots__/prompt.spec.js.snap @@ -32,7 +32,6 @@ How MainChar speaks: SAMPLECHAT MainChar MainChar: FIRST -MainChar: ORIGINAL MainChar:" `; @@ -99,7 +98,6 @@ Scenario: MAIN MainChar This is how OtherBot should talk: SAMPLECHAT OtherBot MainChar: FIRST ChatOwner: SECOND -MainChar: ORIGINAL OtherBot:" `; @@ -165,7 +163,6 @@ SAMPLECHAT OtherBot System: New conversation started. Previous conversations are examples only. MainChar: FIRST ChatOwner: SECOND -MainChar: ORIGINAL OtherBot:" `; From 8adb1201414e9450d4ba6beec504c90bccc49be5 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Tue, 9 Apr 2024 01:50:44 +0300 Subject: [PATCH 4/6] fix: cut prompt to history's last message --- common/prompt.ts | 2 +- common/template-parser.ts | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/common/prompt.ts b/common/prompt.ts index d2c9f104a..02c2b44dd 100644 --- a/common/prompt.ts +++ b/common/prompt.ts @@ -510,7 +510,7 @@ function createPostPrompt( if (opts.kind === 'chat-query') { post.push(`Query Response:`) - } else if (opts.kind !== 'continue') { + } else { post.push(`${opts.replyAs.name}:`) } diff --git a/common/template-parser.ts b/common/template-parser.ts index f8e0a423d..c446fa1a8 100644 --- a/common/template-parser.ts +++ b/common/template-parser.ts @@ -138,16 +138,22 @@ export async function parseTemplate( parts.ujb = render(parts.ujb, opts) } - const ast = parser.parse(template, {}) as PNode[] + let ast = parser.parse(template, {}) as PNode[] - // hack: when we continue, remove the post along with the last newline from tree - if (opts.continue && ast.length > 1) { - const last = ast[ast.length - 1] - if (typeof last !== 'string' && last.kind === 'placeholder' && last.value === 'post') { - ast.pop() - } - if (ast[ast.length - 1] === '\n') { - ast.pop() + /** + * Continuing the previous message: + * In this case our goal is to end the prompt as close to the + * last message as possible. + */ + if (opts.continue) { + const historyIndex = ast.findIndex( + (node) => + typeof node !== 'string' && + ((node.kind === 'placeholder' && node.value === 'history') || + (node.kind === 'each' && node.value === 'history')) + ) + if (historyIndex !== -1) { + ast = ast.slice(0, historyIndex + 1) } } @@ -447,7 +453,7 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat let i = 0 for (const entity of entities) { let curr = '' - for (const child of children) { + children_loop: for (const child of children) { if (typeof child === 'string') { curr += child continue @@ -473,6 +479,8 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat case 'history-prop': { const result = renderProp(child, opts, entity, i) if (result) curr += result + // when continuing, cut the first node (last response) to its message + if (opts.continue && i === 0 && isHistory && child.prop === 'message') break children_loop break } @@ -482,6 +490,8 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat if (!prop) break const result = renderEntityCondition(child.children, opts, entity, i) curr += result + // when continuing, cut the first node (last response) to its message + if (opts.continue && i === 0 && isHistory) break children_loop break } } From 69a011bd02e29c1545f34d3180dffea48c43865c Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Thu, 2 May 2024 04:44:43 +0300 Subject: [PATCH 5/6] feat: always put {{history}} when continue --- common/template-parser.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/template-parser.ts b/common/template-parser.ts index c446fa1a8..e77b46b3b 100644 --- a/common/template-parser.ts +++ b/common/template-parser.ts @@ -153,6 +153,11 @@ export async function parseTemplate( (node.kind === 'each' && node.value === 'history')) ) if (historyIndex !== -1) { + const node = ast[historyIndex] as PlaceHolder | IteratorNode; + // replace iterator with normalized history + if (node.kind === 'each' && node.value === 'history') { + ast[historyIndex] = { kind: 'placeholder', value: 'history' } as PlaceHolder + } ast = ast.slice(0, historyIndex + 1) } } @@ -453,7 +458,7 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat let i = 0 for (const entity of entities) { let curr = '' - children_loop: for (const child of children) { + for (const child of children) { if (typeof child === 'string') { curr += child continue @@ -479,8 +484,6 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat case 'history-prop': { const result = renderProp(child, opts, entity, i) if (result) curr += result - // when continuing, cut the first node (last response) to its message - if (opts.continue && i === 0 && isHistory && child.prop === 'message') break children_loop break } @@ -490,8 +493,6 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: Templat if (!prop) break const result = renderEntityCondition(child.children, opts, entity, i) curr += result - // when continuing, cut the first node (last response) to its message - if (opts.continue && i === 0 && isHistory) break children_loop break } } From e54b72ef8156872d4f839410dc57f2bcf68d2682 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Thu, 2 May 2024 05:17:48 +0300 Subject: [PATCH 6/6] feat: better sentence concat --- common/template-parser.ts | 2 +- common/util.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/template-parser.ts b/common/template-parser.ts index e77b46b3b..f931e4746 100644 --- a/common/template-parser.ts +++ b/common/template-parser.ts @@ -153,7 +153,7 @@ export async function parseTemplate( (node.kind === 'each' && node.value === 'history')) ) if (historyIndex !== -1) { - const node = ast[historyIndex] as PlaceHolder | IteratorNode; + const node = ast[historyIndex] as PlaceHolder | IteratorNode // replace iterator with normalized history if (node.kind === 'each' && node.value === 'history') { ast[historyIndex] = { kind: 'placeholder', value: 'history' } as PlaceHolder diff --git a/common/util.ts b/common/util.ts index 83e9ae238..24ecc992e 100644 --- a/common/util.ts +++ b/common/util.ts @@ -176,12 +176,12 @@ export function trimSentence(text: string) { } export function concatenateSentence(text: string, next: string) { - if (!text || !next) return text + next - if (text.endsWith('\n') || next.startsWith('\n')) { + if (!text || !next) return `${text}${next}` + if (next.startsWith('\n')) { return `${text.trimEnd()}\n${next.trimStart()}` } - return `${text.trimEnd()} ${next.trimStart()}` + return `${text.trimEnd()}${next}` } export function slugify(str: string) {