diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 344ae4c076f0..83755ffb079d 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -337,6 +337,11 @@ const layer = Layer.effect( }, }) if (outcome.needsContinuation) return true + // Stopping short of the catalog limit means the context window ran out, so continuing alone cannot help. + if (outcome.early && recoverOverflow) { + recoverOverflow = false + yield* compaction.compact({ reason: "overflow", context: loaded }) + } yield* bus.publish(SessionEvent.Synthetic, { sessionID, text: CONTINUE_AFTER_OUTPUT_LIMIT }) assistantMessageID = SessionMessage.ID.create() }), diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 8d1a6c076277..8578ffa6b6c3 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -38,7 +38,7 @@ export type Outcome = Data.TaggedEnum<{ } RecoverFull: {} Compacted: {} - OutputLimit: { readonly needsContinuation: boolean } + OutputLimit: { readonly needsContinuation: boolean; readonly early: boolean } }> export const Outcome = Data.taggedEnum() @@ -277,7 +277,11 @@ export const make = Effect.gen(function* () { if (tools.interrupted && Exit.isFailure(joined)) return yield* Effect.failCause(joined.cause) if (record.failure) return yield* new StepFailedError({ error: record.failure }) const needsContinuation = input.prepared.request.toolChoice?.type !== "none" && record.needsContinuation - if (record.finish?.finish === "length") return Outcome.OutputLimit({ needsContinuation }) + if (record.finish?.finish === "length") { + const output = record.finish.usage?.outputTokens + const early = output !== undefined && input.model.limit.output > 0 && output < input.model.limit.output + return Outcome.OutputLimit({ needsContinuation, early }) + } return Outcome.Completed({ needsContinuation }) }), ) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 3ccc2518191e..c8ed39ea1216 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -5732,6 +5732,42 @@ describe("SessionRunnerLLM", () => { }) } + scenario("compacts before continuing only when a length stop ends below the output limit", function* (s) { + yield* setupOverflowRecovery(s) + const truncated = (outputTokens: number) => + TestLLM.complete( + { reason: { normalized: "length" }, usage: { outputTokens } }, + LLMEvent.textStart({ id: "partial" }), + LLMEvent.textDelta({ id: "partial", text: "Partial" }), + LLMEvent.textEnd({ id: "partial" }), + ) + yield* s.llm.push( + truncated(200), + TestLLM.text("## Objective\n- Early stop", "text-summary"), + TestLLM.text("Finished", "text-final"), + ) + yield* s.runPrompt("Continue") + + expect(s.requests).toHaveLength(3) + expect(userTexts(s.requests[1]).at(-1)).toContain("## Objective") + expect(userTexts(s.requests[2])[0]).toContain("\n## Objective\n- Early stop\n") + expect(userTexts(s.requests[2]).at(-1)).toContain("output token limit") + expect(yield* s.context).toMatchObject([ + { type: "compaction" }, + { type: "synthetic" }, + { type: "assistant", finish: "stop" }, + ]) + expect((yield* s.messages).some((message) => message.type === "assistant" && message.finish === "length")).toBe( + true, + ) + + s.requests.length = 0 + yield* s.llm.push(truncated(1_000), TestLLM.text("Finished", "text-full")) + yield* s.runPrompt("Continue again") + expect(s.requests).toHaveLength(2) + expect(userTexts(s.requests[1]).at(-1)).toContain("output token limit") + }) + for (const tools of ["hosted", "unfinished"] as const) { scenario(`caps output token limit continuations and resets for new input (${tools})`, function* (s) { const toolEvents =