Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions agents/src/voice/agent_session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import {
type CloseEvent,
CloseReason,
type ConversationItemAddedEvent,
type CustomEvent,
type ErrorEvent,
type FunctionToolsExecutedEvent,
type MetricsCollectedEvent,
Expand Down Expand Up @@ -157,6 +158,7 @@ export type AgentSessionCallbacks = {
[AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;
[AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;
[AgentSessionEventTypes.OverlappingSpeech]: (ev: OverlappingSpeechEvent) => void;
[AgentSessionEventTypes.CustomEvent]: (ev: CustomEvent) => void;
};

export type AgentSessionOptions<UserData = UnknownUserData> = {
Expand Down
22 changes: 22 additions & 0 deletions agents/src/voice/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export enum AgentSessionEventTypes {
SpeechCreated = 'speech_created',
AgentFalseInterruption = 'agent_false_interruption',
OverlappingSpeech = 'overlapping_speech',
CustomEvent = 'custom_event',
Error = 'error',
Close = 'close',
}
Expand Down Expand Up @@ -340,6 +341,26 @@ export const createAgentFalseInterruptionEvent = ({
createdAt,
});

export type CustomEvent = {
type: 'custom_event';
/** Maps to proto `CustomEvent.type`. */
eventType: string;
/** Maps to proto `CustomEvent.payload` (arbitrary JSON object). */
payload: Record<string, unknown>;
createdAt: number;
};

export const createCustomEvent = (
eventType: string,
payload: Record<string, unknown> = {},
createdAt: number = Date.now(),
): CustomEvent => ({
type: 'custom_event',
eventType,
payload,
createdAt,
});

export type AgentEvent =
| UserInputTranscribedEvent
| UserStateChangedEvent
Expand All @@ -351,5 +372,6 @@ export type AgentEvent =
| SpeechCreatedEvent
| AgentFalseInterruptionEvent
| OverlappingSpeechEvent
| CustomEvent
| ErrorEvent
| CloseEvent;
37 changes: 36 additions & 1 deletion agents/src/voice/remote_session.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
//
// SPDX-License-Identifier: Apache-2.0
import { Duration, Timestamp } from '@bufbuild/protobuf';
import { Duration, Struct, Timestamp } from '@bufbuild/protobuf';
import { AgentSession as pb } from '@livekit/protocol';
import type { ByteStreamReader, Room, TextStreamInfo } from '@livekit/rtc-node';
import { ThrowsPromise } from '@livekit/throws-transformer/throws';
Expand Down Expand Up @@ -32,6 +32,7 @@ import {
type AgentState,
type AgentStateChangedEvent,
type ConversationItemAddedEvent,
type CustomEvent,
type ErrorEvent,
type FunctionToolsExecutedEvent,
type MetricsCollectedEvent,
Expand Down Expand Up @@ -62,6 +63,7 @@ export type RemoteSessionEventTypes =
| 'function_tools_executed'
| 'overlapping_speech'
| 'amd_prediction'
| 'custom_event'
| 'session_usage'
| 'error';

Expand All @@ -74,10 +76,22 @@ export type RemoteSessionCallbacks = {
function_tools_executed: (ev: pb.AgentSessionEvent_FunctionToolsExecuted) => void;
overlapping_speech: (ev: pb.AgentSessionEvent_OverlappingSpeech) => void;
amd_prediction: (ev: pb.AgentSessionEvent_AmdPrediction) => void;
custom_event: (ev: { eventType: string; payload: Record<string, unknown> }) => void;
session_usage: (ev: pb.AgentSessionEvent_SessionUsageUpdated) => void;
error: (ev: pb.AgentSessionEvent_Error) => void;
};

function dictToStruct(data: Record<string, unknown>): Struct {
return Struct.fromJson(data);
Comment on lines +84 to +85
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Validate custom-event payload before Struct conversion

dictToStruct() passes Record<string, unknown> directly into Struct.fromJson(), but this API only accepts JSON-compatible values. Because the new public payload type permits non-JSON data (for example { foo: undefined }, Date, bigint, or functions), emitting a custom event can throw synchronously in onCustomEvent, which can break event forwarding for the session at runtime. Tightening the type to a JSON object (or sanitizing/rejecting unsupported values before conversion) avoids this runtime failure mode.

Useful? React with 👍 / 👎.

}

function structToDict(st: Struct | undefined): Record<string, unknown> {
if (!st) {
return {};
}
return st.toJson() as Record<string, unknown>;
}

// ===========================================================================
// SessionTransport
// ===========================================================================
Expand Down Expand Up @@ -522,6 +536,7 @@ export class SessionHost {
session.on(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected);
session.on(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech);
session.on(AgentSessionEventTypes.Error, this.onHostError);
session.on(AgentSessionEventTypes.CustomEvent, this.onCustomEvent);
}
}

Expand Down Expand Up @@ -550,6 +565,7 @@ export class SessionHost {
this.session.off(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected);
this.session.off(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech);
this.session.off(AgentSessionEventTypes.Error, this.onHostError);
this.session.off(AgentSessionEventTypes.CustomEvent, this.onCustomEvent);
}

if (this.recvTask) {
Expand Down Expand Up @@ -713,6 +729,19 @@ export class SessionHost {
);
};

private onCustomEvent = (event: CustomEvent): void => {
this.emitEvent(
{
case: 'customEvent',
value: new pb.CustomEvent({
type: event.eventType,
payload: dictToStruct(event.payload),
}),
},
event.createdAt,
);
};

/**
* @internal — forwards an AMD prediction to the connected
* {@link RemoteSession} peer. Mirrors python
Expand Down Expand Up @@ -1007,6 +1036,12 @@ export class RemoteSession extends (EventEmitter as new () => TypedEventEmitter<
case 'error':
this.emit('error', ev.value);
break;
case 'customEvent':
this.emit('custom_event', {
eventType: ev.value.type,
payload: structToDict(ev.value.payload),
});
break;
}
}

Expand Down
8 changes: 7 additions & 1 deletion examples/src/multi_agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,10 @@ export default defineAgent({
},
});

cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));
cli.runApp(
new ServerOptions({
agent: fileURLToPath(import.meta.url),
// Enables explicit dispatch so agents-cli can summon this worker by name.
agentName: 'multi-agent-js',
}),
);
Loading