Skip to content

Prebuilt: carry TML Inkling on b10173 for the nightly PR set #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
danielhanchen wants to merge 5 commits into from
Closed

Prebuilt: carry TML Inkling on b10173 for the nightly PR set #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ce16fff
Add TML Inkling architecture
danielhanchen Jul 18, 2026
453c438
Merge remote-tracking branch 'origin/master' into pr-25731-head
danielhanchen Jul 23, 2026
e38da9a
Merge remote-tracking branch 'upstream/master' into add-inkling-fix
shimmyshimmer Jul 26, 2026
d8f8750
inkling : adapt to the renamed thinking_end_tags chat API
shimmyshimmer Jul 26, 2026
3af940f
Merge TML Inkling (#25731) onto b10173 for the nightly prebuild
danielhanchen Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions common/chat-peg-parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,7 +594,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
const std::string & args_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key) {
const std::string & gen_call_id_key,
bool require_object_args) {

auto tool_choices = choice();

Expand Down Expand Up @@ -631,10 +632,10 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
// Arguments — either wrapped in args_key or parsed directly
common_peg_parser args_parser = eps();
if (args_key.empty()) {
args_parser = tool_args(schema(json(), "tool-" + name + "-schema", params));
args_parser = tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));
} else {
args_parser = literal("\"" + effective_args_key + "\"") + space() + literal(":") + space() +
tool_args(schema(json(), "tool-" + name + "-schema", params));
tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));
}
inner_fields.push_back(args_parser);

Expand Down Expand Up @@ -673,7 +674,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key) {
const std::string & gen_call_id_key,
bool require_object_args) {

auto tool_choices = choice();

Expand All @@ -695,7 +697,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
auto nested_args = literal("\"" + nested_args_field + "\"") + space() + literal(":") + space() +
tool_args(schema(json(), "tool-" + name + "-schema", params));
tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));

auto nested_object = literal("{") + space() +
nested_name + space() + literal(",") + space() +
Expand Down Expand Up @@ -747,7 +749,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
const std::string & call_id_key,
const std::string & gen_call_id_key,
const std::vector<std::string> & parameters_order,
bool accept_openai_wrapper) {
bool accept_openai_wrapper,
bool require_object_args) {

auto tool_choices = choice();
auto name_key_parser = literal("\"" + effective_name_key + "\"");
Expand All @@ -764,7 +767,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
auto tool_name_ = name_key_parser + space() + literal(":") + space() +
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
auto tool_args_ = args_key_parser + space() + literal(":") + space() +
tool_args(schema(json(), "tool-" + name + "-schema", params));
tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));

// Build ID parsers if keys are provided
common_peg_parser id_parser = eps();
Expand Down Expand Up @@ -879,7 +882,8 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(
const std::string & call_id_key,
const std::string & gen_call_id_key,
const std::vector<std::string> & parameters_order,
bool accept_openai_wrapper) {
bool accept_openai_wrapper,
bool require_object_args) {
if (!tools.is_array() || tools.empty()) {
return eps();
}
Expand All @@ -890,14 +894,14 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(
// Dispatch to the appropriate builder based on the JSON layout mode
common_peg_parser tool_choices = eps();
if (function_is_key) {
tool_choices = build_json_tools_function_is_key(tools, args_key, effective_args_key, call_id_key, gen_call_id_key);
tool_choices = build_json_tools_function_is_key(tools, args_key, effective_args_key, call_id_key, gen_call_id_key, require_object_args);
} else {
auto name_spec = parse_key_spec(effective_name_key);
auto args_spec = parse_key_spec(effective_args_key);
if (!name_spec.first.empty() || !args_spec.first.empty()) {
tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key);
tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, require_object_args);
} else {
tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper);
tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper, require_object_args);
}
}

Expand Down
12 changes: 8 additions & 4 deletions common/chat-peg-parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {
const std::string & call_id_key = "",
const std::string & gen_call_id_key = "",
const std::vector<std::string> & parameters_order = {},
bool accept_openai_wrapper = false);
bool accept_openai_wrapper = false,
bool require_object_args = false);

// Legacy-compatible helper for building XML/tagged style tool calls
// Used by tests and manual parsers
Expand All @@ -157,21 +158,24 @@ class common_chat_peg_builder : public common_peg_parser_builder {
const std::string & args_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key);
const std::string & gen_call_id_key,
bool require_object_args);

common_peg_parser build_json_tools_nested_keys(const nlohmann::ordered_json & tools,
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key);
const std::string & gen_call_id_key,
bool require_object_args);

common_peg_parser build_json_tools_flat_keys(const nlohmann::ordered_json & tools,
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key,
const std::vector<std::string> & parameters_order,
bool accept_openai_wrapper);
bool accept_openai_wrapper,
bool require_object_args);
};

inline common_peg_arena build_chat_peg_parser(
Expand Down
127 changes: 127 additions & 0 deletions common/chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2530,6 +2530,125 @@ static common_chat_params common_chat_params_init_minimax_m3(const common_chat_t
return data;
}

// Inkling / TML typed-content-block parser: <|end_message|> separates blocks within a turn,
// <|content_model_end_sampling|> is the sole end-of-generation token (mirrors sglang TmlDetector).
static common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;

const std::string MSG_MODEL = "<|message_model|>";
const std::string MSG_USER = "<|message_user|>";
const std::string MSG_SYSTEM = "<|message_system|>";
const std::string MSG_TOOL = "<|message_tool|>";
const std::string THINK = "<|content_thinking|>";
const std::string TEXT = "<|content_text|>";
const std::string END_MESSAGE = "<|end_message|>";
const std::string END_SAMPLING = "<|content_model_end_sampling|>";
const std::string INVOKE_TOOL = "<|content_invoke_tool_json|>";

data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = THINK;
data.thinking_end_tags = {END_MESSAGE};
data.preserved_tokens = {
MSG_MODEL, MSG_USER, MSG_SYSTEM, MSG_TOOL,
THINK, TEXT, END_MESSAGE, END_SAMPLING, INVOKE_TOOL,
};

auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
data.message_delimiters = {
{ COMMON_CHAT_ROLE_ASSISTANT, MSG_MODEL },
{ COMMON_CHAT_ROLE_USER, MSG_USER },
{ COMMON_CHAT_ROLE_SYSTEM, MSG_SYSTEM },
{ COMMON_CHAT_ROLE_TOOL, MSG_TOOL },
};

auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;

if (inputs.has_continuation()) {
const auto & msg = inputs.continue_msg;

data.generation_prompt = MSG_MODEL + THINK + msg.reasoning_content;
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
data.generation_prompt += END_MESSAGE + TEXT + msg.render_content();
}

data.prompt += data.generation_prompt;
}

auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(MSG_MODEL);
auto end = p.end();

// thinking block; may also reappear mid-turn (after content), so it is both an optional
// prefix and a choice inside the block loops. With reasoning_format=NONE keep it
// (markers included) inline as content
common_peg_parser reasoning_block = p.eps();
if (extract_reasoning) {
reasoning_block = p.literal(THINK) +
p.reasoning(p.until_one_of({ END_MESSAGE, TEXT, END_SAMPLING })) +
p.optional(p.literal(END_MESSAGE));
} else {
reasoning_block = p.content(p.literal(THINK) +
p.until_one_of({ END_MESSAGE, TEXT, END_SAMPLING }) +
p.optional(p.literal(END_MESSAGE)));
}
auto reasoning = p.optional(reasoning_block);

// TML re-emits <|message_model|> before each content block; a turn may contain several
// text blocks (one per content part), so the block repeats and bodies concatenate.
// THINK stops the content scan so a mid-turn thinking block is never leaked as text
auto text_block = p.optional(p.literal(MSG_MODEL)) +
p.optional(p.literal(TEXT)) +
p.content(p.until_one_of({ THINK, END_MESSAGE, END_SAMPLING })) +
p.optional(p.literal(END_MESSAGE));
auto text_content = p.one_or_more(p.choice({ reasoning_block, text_block }));

if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
return generation_prompt + reasoning + text_content +
p.optional(p.literal(END_SAMPLING)) + end;
}

// each call is its own block (role opener + bare name echo + JSON section);
// force_tool_calls=true makes the JSON section required so a pure-text answer fails the
// block cleanly; parallel calls are separate blocks, hence repeat + parallel=false
auto tool_section = p.standard_json_tools(
INVOKE_TOOL, END_MESSAGE, inputs.tools, /* parallel_tool_calls = */ false,
/* force_tool_calls = */ true,
/* name_key = */ "name",
/* args_key = */ "args",
/* array_wrapped = */ false,
/* function_is_key = */ false,
/* call_id_key = */ "",
/* gen_call_id_key = */ "",
/* parameters_order = */ {},
/* accept_openai_wrapper = */ false,
/* require_object_args = */ true);
// the name-echo scan must stop at any block marker: a greedy until(INVOKE_TOOL) returns
// NEED_MORE_INPUT mid-stream, which choice() treats as a match and shadows the text branch
auto tool_block = p.optional(p.literal(MSG_MODEL)) +
p.until_one_of({ INVOKE_TOOL, TEXT, THINK, END_MESSAGE, END_SAMPLING }) +
tool_section;
auto tool_calls = inputs.parallel_tool_calls ? p.one_or_more(tool_block) : tool_block;
// turns may interleave narration, thinking and calls; parse block-by-block (tool block
// first) since a whole-body choice would let the text branch swallow tool blocks into
// visible content
auto mixed_body = p.one_or_more(p.choice({ tool_block, reasoning_block, text_block }));
auto body = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED
? tool_calls
: mixed_body;

return generation_prompt + reasoning + body +
p.optional(p.literal(END_SAMPLING)) + end;
});

data.parser = parser.save();

return data;
}

namespace workaround {

static void map_developer_role_to_system(json & messages) {
Expand Down Expand Up @@ -2947,6 +3066,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_cohere2moe(tmpl, params);
}

// Inkling / TML: this marker combination is unique to the template
if (src.find("<|content_thinking|>") != std::string::npos &&
src.find("<|content_text|>") != std::string::npos &&
src.find("<|message_model|>") != std::string::npos) {
LOG_DBG("Using specialized template: Inkling\n");
return common_chat_params_init_inkling(tmpl, params);
}

if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);
Expand Down
2 changes: 2 additions & 0 deletions conversion/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@
"HunYuanVLForConditionalGeneration": "hunyuan",
"HYV3ForCausalLM": "hunyuan",
"IQuestCoderForCausalLM": "llama",
"InklingForConditionalGeneration": "inkling",
"InternLM2ForCausalLM": "internlm",
"InternLM3ForCausalLM": "internlm",
"JAISLMHeadModel": "jais",
Expand Down Expand Up @@ -279,6 +280,7 @@
"GraniteSpeechPlusForConditionalGeneration": "granite",
"HunYuanVLForConditionalGeneration": "hunyuan",
"Idefics3ForConditionalGeneration": "smolvlm",
"InklingForConditionalGeneration": "inkling",
"InternVisionModel": "internvl",
"JanusForConditionalGeneration": "januspro",
"KimiK25ForConditionalGeneration": "kimivl",
Expand Down
Loading
Loading