Add optional C++ rendering for HF chat templates

.github/workflows/pull.yml

@@ -24,7 +24,7 @@ jobs:
       timeout: 90
       script: |
         set -ex
-        cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test
+        cmake -DCMAKE_BUILD_TYPE=Debug -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON test -Bbuild/test
         cmake --build build/test -j9 --config Debug
         pushd build/test && ctest && popd
 
@@ -54,7 +54,7 @@ jobs:
           \$PSNativeCommandUseErrorActionPreference = \$true
 
           # Run C++ unit tests
-          cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test -T ClangCL
+          cmake -DCMAKE_BUILD_TYPE=Debug -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON test -Bbuild/test -T ClangCL
           cmake --build build/test -j9 --config Debug
           if (\$LASTEXITCODE -ne 0) {
             Write-Host "Build was not successful. Exit code: \$LASTEXITCODE."

.github/workflows/trunk.yml

@@ -31,7 +31,7 @@ jobs:
       timeout: 90
       script: |
         set -ex
-        cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test
+        cmake -DCMAKE_BUILD_TYPE=Debug -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON test -Bbuild/test
         cmake --build build/test -j9 --config Debug
         pushd build/test && ctest && popd
 

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "third-party/pcre2"]
 	path = third-party/pcre2
 	url = https://github.com/PCRE2Project/pcre2.git
+[submodule "third-party/minja"]
+	path = third-party/minja
+	url = https://github.com/google/minja.git

CMakeLists.txt

@@ -19,6 +19,7 @@ project(Tokenizers)
 option(TOKENIZERS_BUILD_TEST "Build tests" OFF)
 option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF)
 option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF)
+option(TOKENIZERS_BUILD_CHAT_TEMPLATE "Build HF Jinja chat template rendering" OFF)
 option(SUPPORT_REGEX_LOOKAHEAD
        "Support regex lookahead patterns (requires PCRE2)" OFF
 )
@@ -158,6 +159,9 @@ endif()
 # Build tools
 if(TOKENIZERS_BUILD_TOOLS)
   add_subdirectory(examples/tokenize_tool)
+  if(TOKENIZERS_BUILD_CHAT_TEMPLATE)
+    add_subdirectory(examples/chat_template_tool)
+  endif()
 endif()
 
 # Build Python bindings
@@ -193,6 +197,27 @@ endif()
 # Installation rules
 include(GNUInstallDirs)
 
+if(TOKENIZERS_BUILD_CHAT_TEMPLATE)
+  if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third-party/minja/include/minja/minja.hpp")
+    message(FATAL_ERROR "Initialise third-party/minja with git submodule update --init")
+  endif()
+  add_library(tokenizers_chat_template STATIC src/chat_template.cpp)
+  add_library(tokenizers::chat_template ALIAS tokenizers_chat_template)
+  set_target_properties(tokenizers_chat_template PROPERTIES
+    EXPORT_NAME chat_template POSITION_INDEPENDENT_CODE ON
+  )
+  target_compile_features(tokenizers_chat_template PUBLIC cxx_std_17)
+  target_include_directories(tokenizers_chat_template
+    PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+           $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/json/single_include>
+           $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third-party/minja/include
+  )
+  install(TARGETS tokenizers_chat_template EXPORT tokenizers-targets
+          ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+endif()
+
 if(NOT TOKENIZERS_BUILD_PYTHON)
   # Install the library and its dependencies
   install(

README.md

@@ -18,6 +18,11 @@ Adapted from https://github.com/sewenew/tokenizer.
 ## Huggingface tokenizer
 Compatible with https://github.com/huggingface/tokenizers/.
 
+Optional C++ chat template rendering is available with
+`TOKENIZERS_BUILD_CHAT_TEMPLATE=ON`. See the
+[chat template example](examples/chat_template_tool/README.md) for loading HF
+templates and rendering prompts separately from tokenization and generation.
+
 ## Llama2.c tokenizer
 Adapted from https://github.com/karpathy/llama2.c.
 

examples/chat_template_tool/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_executable(chat_template_tool main.cpp)
+target_link_libraries(chat_template_tool PRIVATE tokenizers::chat_template)

examples/chat_template_tool/README.md

@@ -0,0 +1,61 @@
+# HF chat template rendering
+
+Enable `TOKENIZERS_BUILD_CHAT_TEMPLATE` and `TOKENIZERS_BUILD_TOOLS`, then build
+`chat_template_tool`. This optional target uses the pinned Minja Jinja renderer;
+the default tokenizer build does not compile it.
+
+```bash
+cmake -S . -B build -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON -DTOKENIZERS_BUILD_TOOLS=ON
+cmake --build build --target chat_template_tool
+build/examples/chat_template_tool/chat_template_tool model_dir messages.json kwargs.json
+```
+
+`messages.json` uses HF's role/content conversation structure, for example:
+
+```json
+[{"role": "user", "content": "Give me a short introduction to large language models."}]
+```
+
+`kwargs.json` can contain `{"enable_thinking": false}` or other variables used by
+the model template. A final `0` or `1` argument controls `add_generation_prompt`
+(default `1`). Output is the rendered prompt, with no additional newline.
+
+C++ runners can link `tokenizers::chat_template` and use the same API:
+
+```cpp
+tokenizers::HFChatTemplate renderer;
+if (renderer.load(model_directory) != tokenizers::Error::Ok) {
+  // Report a missing/unsupported template or choose an explicit app fallback.
+}
+auto prompt = renderer.apply(messages, true, {{"enable_thinking", false}});
+if (prompt.ok()) {
+  auto ids = tokenizer.encode(*prompt, 0, 0);
+}
+```
+
+Rendering is separate from tokenization and generation. Encode without adding
+BOS/EOS again because the model's template supplies the required special tokens.
+`enable_thinking` changes only templates that reference that variable.
+
+The loader reads `chat_template.jinja`, legacy `tokenizer_config.json` templates
+or legacy processor `chat_template.json`, plus string/object special-token
+metadata. Standalone Jinja overrides legacy templates. Named templates can be
+selected with `load(directory, "tool_use")`; automatic tool-template selection is
+not performed. Named standalone files live in `additional_chat_templates`.
+
+Missing artifacts return `LoadFailure`; malformed/unsupported templates and
+rendering failures return `ParseFailure`. Failed reloads preserve the old template.
+Each artifact is limited to 1 MiB. Load before sharing a renderer; concurrent
+`apply` calls have independent context, but concurrent reloads are not supported.
+Use trusted model templates: Minja implements a subset of Jinja for chat models,
+not every Python/Jinja feature. This first library/example change does not alter
+existing ExecuTorch app prompts or add a Kotlin/JNI binding for this new API.
+The extra `capitalize` filter covers ASCII role names; Unicode case expansion is
+unsupported and returns a render error. Multimodal messages must use the content
+block structure expected by their processor template.
+
+Fixtures pin Qwen3 and SmolVLM templates and contain exact reference output from
+Transformers, covering Unicode, multiple turns, image content, generation prompts
+and thinking enabled/disabled/default. Source revisions and hashes are in
+`test/resources/chat_templates/manifest.json`. Fixture templates are Apache-2.0;
+Minja is MIT. The normal project tests include the renderer when the option is on.

examples/chat_template_tool/main.cpp

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+#include <pytorch/tokenizers/chat_template.h>
+#include <fstream>
+#include <iostream>
+
+int main(int argc, char** argv) {
+  if (argc < 3 || argc > 5) {
+    std::cerr
+        << "Usage: chat_template_tool MODEL_DIR MESSAGES_JSON [KWARGS_JSON] [0|1 add_generation_prompt]\n";
+    return 1;
+  }
+  try {
+    std::ifstream messages_file(argv[2]);
+    const auto messages = nlohmann::ordered_json::parse(messages_file);
+    auto kwargs = nlohmann::ordered_json::object();
+    if (argc >= 4) {
+      std::ifstream kwargs_file(argv[3]);
+      kwargs = nlohmann::ordered_json::parse(kwargs_file);
+    }
+    if (argc == 5 && std::string(argv[4]) != "0" &&
+        std::string(argv[4]) != "1") {
+      throw std::runtime_error("Generation prompt must be 0 or 1");
+    }
+    tokenizers::HFChatTemplate renderer;
+    if (renderer.load(argv[1]) != tokenizers::Error::Ok) {
+      throw std::runtime_error("Cannot load HF chat template");
+    }
+    const auto prompt = renderer.apply(
+        messages, argc < 5 || std::string(argv[4]) == "1", kwargs);
+    if (!prompt.ok()) {
+      throw std::runtime_error("Cannot render HF chat template");
+    }
+    std::cout << *prompt;
+    return 0;
+  } catch (const std::exception& error) {
+    std::cerr << error.what() << '\n';
+    return 1;
+  }
+}

include/pytorch/tokenizers/chat_template.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+#pragma once
+
+#include <nlohmann/json.hpp>
+#include <pytorch/tokenizers/result.h>
+#include <memory>
+#include <string>
+
+namespace tokenizers {
+
+/** Render HF chat templates without loading model weights or generating text.
+ */
+class HFChatTemplate {
+ public:
+  HFChatTemplate();
+  ~HFChatTemplate();
+  HFChatTemplate(const HFChatTemplate&) = delete;
+  HFChatTemplate& operator=(const HFChatTemplate&) = delete;
+
+  /**
+   * Load a model directory, tokenizer_config.json, chat_template.json or Jinja.
+   * Standalone Jinja takes precedence over the config's legacy template.
+   * Select a named template explicitly; the default name is "default".
+   * A failed reload preserves the previously loaded template.
+   */
+  Error load(const std::string& path, const std::string& name = "default");
+
+  /**
+   * Equivalent to rendering apply_chat_template(..., tokenize=False).
+   * kwargs can include enable_thinking, tools and other model-specific values.
+   * messages and add_generation_prompt are supplied by the dedicated arguments.
+   * Encode the returned text with BOS/EOS counts zero to avoid duplicate
+   * tokens.
+   */
+  Result<std::string> apply(
+      const nlohmann::ordered_json& messages,
+      bool add_generation_prompt = true,
+      const nlohmann::ordered_json& kwargs =
+          nlohmann::ordered_json::object()) const;
+
+  bool is_loaded() const;
+
+ private:
+  struct Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+} // namespace tokenizers