From ca59bc665ff6d49a30ecc0754a6df8a4f6d0dd62 Mon Sep 17 00:00:00 2001 From: Tony Powell Date: Thu, 8 Oct 2026 11:53:41 -0400 Subject: [PATCH 01/14] feat(playground): add a decision model mode to the playground Adds a Decision tab to the model picker on the chat playground. Selecting a decision model swaps the chat template for a state + typed questions editor (Choice, Noul/Predicate, Score) and runs the `createDecision` mutation instead of the streaming chat subscription. - Model picker: LLM / Decision tabs backed by `playgroundModels(modelType: DECISION)`; only providers with chat models count as "provisioned" so a decision-only key does not change the LLM provider list. - Decision editor with JSON or text state, per-question criteria, validation before run, and an optional compatible base URL. - Non-streaming runner with repetitions, stale-run protection, inline errors that never echo credentials, named-answer summary with confidence and provider-reported usage, raw response, and trace link. - Switching Decision -> LLM restores the stashed chat config. Decision selection is disabled in dataset mode and on evaluator/prompt surfaces. - Default decision provider and model come from the server catalog, so new decision providers need no frontend change. Agent sessions reject decision models by model type. Co-Authored-By: Claude Fable 5.1 --- js/app/src/Routes.tsx | 4 +- .../src/components/agent/agentSessionModel.ts | 5 + .../src/components/generative/ModelMenu.tsx | 52 +++- .../useModelMenuDataQuery.graphql.ts | 62 +++- .../components/generative/useModelMenuData.ts | 62 +++- .../playground/model/PlaygroundModelMenu.tsx | 80 ++++- js/app/src/constants/decisionConstants.ts | 28 ++ .../src/pages/playground/DecisionResult.tsx | 90 ++++++ js/app/src/pages/playground/Playground.tsx | 78 ++++- .../playground/PlaygroundConfigButton.tsx | 12 +- .../playground/PlaygroundDecisionOutput.tsx | 235 +++++++++++++++ .../playground/PlaygroundDecisionTemplate.tsx | 275 ++++++++++++++++++ .../pages/playground/PlaygroundRunButton.tsx | 12 +- .../pages/playground/PlaygroundTemplate.tsx | 25 +- .../pages/playground/RunMetadataFooter.tsx | 22 +- ...laygroundDecisionOutputMutation.graphql.ts | 194 ++++++++++++ .../__generated__/PlaygroundQuery.graphql.ts | 58 +++- .../PlaygroundDecisionOutput.test.tsx | 236 +++++++++++++++ .../__tests__/decisionUtils.test.ts | 97 ++++++ js/app/src/pages/playground/decisionUtils.ts | 100 +++++++ js/app/src/pages/playground/types.ts | 1 + .../__tests__/playgroundStore.test.ts | 23 ++ .../src/store/playground/playgroundStore.tsx | 15 + js/app/src/store/playground/types.ts | 15 + 24 files changed, 1703 insertions(+), 78 deletions(-) create mode 100644 js/app/src/constants/decisionConstants.ts create mode 100644 js/app/src/pages/playground/DecisionResult.tsx create mode 100644 js/app/src/pages/playground/PlaygroundDecisionOutput.tsx create mode 100644 js/app/src/pages/playground/PlaygroundDecisionTemplate.tsx create mode 100644 js/app/src/pages/playground/__generated__/PlaygroundDecisionOutputMutation.graphql.ts create mode 100644 js/app/src/pages/playground/__tests__/PlaygroundDecisionOutput.test.tsx create mode 100644 js/app/src/pages/playground/__tests__/decisionUtils.test.ts create mode 100644 js/app/src/pages/playground/decisionUtils.ts diff --git a/js/app/src/Routes.tsx b/js/app/src/Routes.tsx index 73893119464..1b6ad05ed78 100644 --- a/js/app/src/Routes.tsx +++ b/js/app/src/Routes.tsx @@ -723,7 +723,7 @@ export const appRouteObjects = createRoutesFromElements( agentRoute: { label: "Playground", description: - "Experiment in the prompt playground with prompts, models, variables, and prompt runs. Supports experimentId, datasetId, splitId, exampleId, promptId, promptVersionId, promptTagName, and selectedSpanNodeId query params.", + "Test LLM prompts or decision models with state and Choice, Noul/Predicate, and Score questions. Supports modelType=DECISION, decisionProvider, decisionModel, experimentId, datasetId, splitId, exampleId, promptId, promptVersionId, promptTagName, and selectedSpanNodeId query params.", }, navigation: { section: "Pages", @@ -741,7 +741,7 @@ export const appRouteObjects = createRoutesFromElements( agentRoute: { label: "Playground", description: - "Experiment in the prompt playground with prompts, models, variables, and prompt runs. Supports experimentId, datasetId, splitId, exampleId, promptId, promptVersionId, promptTagName, and selectedSpanNodeId query params.", + "Test LLM prompts or decision models with state and Choice, Noul/Predicate, and Score questions. Supports modelType=DECISION, decisionProvider, decisionModel, experimentId, datasetId, splitId, exampleId, promptId, promptVersionId, promptTagName, and selectedSpanNodeId query params.", }, }} /> diff --git a/js/app/src/components/agent/agentSessionModel.ts b/js/app/src/components/agent/agentSessionModel.ts index 9ee959fc7cf..c49d815919e 100644 --- a/js/app/src/components/agent/agentSessionModel.ts +++ b/js/app/src/components/agent/agentSessionModel.ts @@ -110,6 +110,11 @@ export function resolvePersistedAgentModel({ export function toAgentModelSelection( config: ModelConfig ): AgentModelSelection { + // Keyed on model type, not provider: a provider may serve both chat and + // decision models, and new decision providers must not need a change here. + if (config.modelType === "DECISION") { + throw new Error("Decision models cannot be used as chat agents"); + } if (config.customProvider) { return { providerType: "custom", diff --git a/js/app/src/components/generative/ModelMenu.tsx b/js/app/src/components/generative/ModelMenu.tsx index 6944d5c469a..11346f991fd 100644 --- a/js/app/src/components/generative/ModelMenu.tsx +++ b/js/app/src/components/generative/ModelMenu.tsx @@ -23,6 +23,9 @@ import { MenuTrigger, SearchField, Text, + Tabs, + TabList, + Tab, useFilter, } from "@phoenix/components"; import { CompactEmptyState } from "@phoenix/components/core/empty"; @@ -37,6 +40,7 @@ import { type GenerativeProviderKey, type ModelCredentialSource, type ModelProviderInfo, + type ModelType, useModelMenuData, } from "@phoenix/components/generative/useModelMenuData"; import { usePreferencesContext } from "@phoenix/contexts"; @@ -58,6 +62,7 @@ export type CustomProviderRef = { export type ModelMenuValue = { provider: GenerativeProviderKey; modelName: string; + modelType?: ModelType; /** * Reference to custom provider if using one */ @@ -99,6 +104,10 @@ type LeadingItemsProps = { */ const KEY_DELIMITER = "\uE000"; +function getModelType(value: ModelMenuValue | null | undefined): ModelType { + return value?.modelType ?? "LLM"; +} + type BuiltinModelInfo = { type: "builtin"; providerKey: string; @@ -198,6 +207,8 @@ export type ModelMenuProps = Pick & * @default "any" */ credentialSource?: ModelCredentialSource; + /** Enable decision selection only on surfaces with a decision execution path. */ + supportsDecisionModels?: boolean; }; export function ModelMenu({ @@ -211,9 +222,11 @@ export function ModelMenu({ selectedLeadingItemId, onLeadingItemSelect, credentialSource, + supportsDecisionModels = false, }: ModelMenuProps) { const { contains } = useFilter({ sensitivity: "base" }); const [searchValue, setSearchValue] = useState(""); + const [modelType, setModelType] = useState(getModelType(value)); const awsBedrockModelPrefix = usePreferencesContext( (state) => state.awsBedrockModelPrefix ); @@ -229,17 +242,19 @@ export function ModelMenu({ }), }); } else { - onChange?.(model); + onChange?.({ ...model, modelType }); } }, - [onChange, awsBedrockModelPrefix] + [onChange, awsBedrockModelPrefix, modelType] ); const { customProviders, modelsByProvider, providerInfoMap, visibleProviders, - } = useModelMenuData({ credentialSource }); + } = useModelMenuData({ credentialSource, modelType }); + const selectableCustomProviders = + modelType === "DECISION" ? [] : customProviders; // Providers whose models are searchable: visible in the menu and with // server dependencies installed, so search never surfaces a model that @@ -282,7 +297,7 @@ export function ModelMenu({ return []; } - return customProviders + return (modelType === "DECISION" ? [] : customProviders) .map((provider) => ({ ...provider, modelNames: provider.modelNames.filter((model) => @@ -290,7 +305,7 @@ export function ModelMenu({ ), })) .filter((provider) => provider.modelNames.length > 0); - }, [searchValue, customProviders, contains]); + }, [searchValue, modelType, customProviders, contains]); const isSearching = searchValue.trim().length > 0; @@ -319,7 +334,14 @@ export function ModelMenu({ : "Select model"; return ( - + { + if (isOpen) { + setModelType(getModelType(value)); + setSearchValue(""); + } + }} + > + ))} + + ) : null} + {runId != null ? : null} + {selected?.error ? ( +
+ {selected.error.message} +
+ ) : null} + {typeof selected?.output === "string" ? ( + + ) : !selected?.error && runId == null ? ( + + Run the decision model to see named answers, confidence, and + usage. + + ) : null} + + + {selected?.spanId ? ( + + + + ) : null} + + ); +} diff --git a/js/app/src/pages/playground/PlaygroundDecisionTemplate.tsx b/js/app/src/pages/playground/PlaygroundDecisionTemplate.tsx new file mode 100644 index 00000000000..96b47f37926 --- /dev/null +++ b/js/app/src/pages/playground/PlaygroundDecisionTemplate.tsx @@ -0,0 +1,275 @@ +import { useSearchParams } from "react-router"; + +import { + Alert, + Button, + Card, + ComboBox, + ComboBoxItem, + Flex, + Input, + Label, + Text, + TextArea, + TextField, + View, +} from "@phoenix/components"; +import { + DEFAULT_CHOICE_CRITERIA, + DEFAULT_NOUL_CRITERIA, + DEFAULT_SCORE_CRITERIA, +} from "@phoenix/constants/decisionConstants"; +import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; + +import { + createDecisionDraft, + getDecisionValidationError, + type DecisionDraft, + type DecisionQuestionDraft, +} from "./decisionUtils"; +import type { PlaygroundInstanceProps } from "./types"; + +/** State and typed questions replace chat messages for decision models. */ +export function PlaygroundDecisionTemplate({ + playgroundInstanceId, +}: PlaygroundInstanceProps) { + const instance = usePlaygroundContext((state) => + state.instances.find((item) => item.id === playgroundInstanceId) + ); + const updateInstance = usePlaygroundContext((state) => state.updateInstance); + const [, setSearchParams] = useSearchParams(); + if (!instance) return null; + const draft = instance.decision ?? createDecisionDraft(); + const isRunning = instance.activeRunId != null; + const error = getDecisionValidationError(draft); + function updateDraft(patch: Partial) { + updateInstance({ + instanceId: playgroundInstanceId, + dirty: true, + patch: { decision: { ...draft, ...patch } }, + }); + } + function updateQuestion({ + id, + patch, + }: { + id: string; + patch: Partial; + }) { + updateDraft({ + questions: draft.questions.map((question) => + question.id === id ? { ...question, ...patch } : question + ), + }); + } + return ( + + + Evaluate state with named questions. Decision models return typed + answers, not a conversation. + + {error ? {error} : null} + + { + updateInstance({ + instanceId: playgroundInstanceId, + dirty: true, + patch: { model: { ...instance.model, modelName } }, + }); + setSearchParams( + (params) => { + params.set("decisionModel", modelName); + return params; + }, + { replace: true } + ); + }} + isDisabled={isRunning} + > + + + + + updateInstance({ + instanceId: playgroundInstanceId, + dirty: true, + patch: { model: { ...instance.model, baseUrl: baseUrl || null } }, + }) + } + isDisabled={isRunning} + > + + + + + + updateDraft({ stateFormat: key === "json" ? "json" : "text" }) + } + isDisabled={isRunning} + > + + Text + + + JSON + + + updateDraft({ state })} + isDisabled={isRunning} + > + +