diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 3b0e2b23..1f308a59 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -24,7 +24,7 @@ jobs: timeout: 90 script: | set -ex - cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test + cmake -DCMAKE_BUILD_TYPE=Debug -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON test -Bbuild/test cmake --build build/test -j9 --config Debug pushd build/test && ctest && popd @@ -54,7 +54,7 @@ jobs: \$PSNativeCommandUseErrorActionPreference = \$true # Run C++ unit tests - cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test -T ClangCL + cmake -DCMAKE_BUILD_TYPE=Debug -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON test -Bbuild/test -T ClangCL cmake --build build/test -j9 --config Debug if (\$LASTEXITCODE -ne 0) { Write-Host "Build was not successful. Exit code: \$LASTEXITCODE." diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 824bbb80..5d860b10 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -31,7 +31,7 @@ jobs: timeout: 90 script: | set -ex - cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test + cmake -DCMAKE_BUILD_TYPE=Debug -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON test -Bbuild/test cmake --build build/test -j9 --config Debug pushd build/test && ctest && popd diff --git a/.gitmodules b/.gitmodules index 04dde04d..2e42fa2b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "third-party/pcre2"] path = third-party/pcre2 url = https://github.com/PCRE2Project/pcre2.git +[submodule "third-party/minja"] + path = third-party/minja + url = https://github.com/google/minja.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 1df99e33..e4f44dce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ project(Tokenizers) option(TOKENIZERS_BUILD_TEST "Build tests" OFF) option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF) option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF) +option(TOKENIZERS_BUILD_CHAT_TEMPLATE "Build HF Jinja chat template rendering" OFF) option(SUPPORT_REGEX_LOOKAHEAD "Support regex lookahead patterns (requires PCRE2)" OFF ) @@ -158,6 +159,9 @@ endif() # Build tools if(TOKENIZERS_BUILD_TOOLS) add_subdirectory(examples/tokenize_tool) + if(TOKENIZERS_BUILD_CHAT_TEMPLATE) + add_subdirectory(examples/chat_template_tool) + endif() endif() # Build Python bindings @@ -193,6 +197,27 @@ endif() # Installation rules include(GNUInstallDirs) +if(TOKENIZERS_BUILD_CHAT_TEMPLATE) + if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third-party/minja/include/minja/minja.hpp") + message(FATAL_ERROR "Initialise third-party/minja with git submodule update --init") + endif() + add_library(tokenizers_chat_template STATIC src/chat_template.cpp) + add_library(tokenizers::chat_template ALIAS tokenizers_chat_template) + set_target_properties(tokenizers_chat_template PROPERTIES + EXPORT_NAME chat_template POSITION_INDEPENDENT_CODE ON + ) + target_compile_features(tokenizers_chat_template PUBLIC cxx_std_17) + target_include_directories(tokenizers_chat_template + PUBLIC $ + $ + $ + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third-party/minja/include + ) + install(TARGETS tokenizers_chat_template EXPORT tokenizers-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif() + if(NOT TOKENIZERS_BUILD_PYTHON) # Install the library and its dependencies install( diff --git a/README.md b/README.md index 96ca2878..2705c53b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ Adapted from https://github.com/sewenew/tokenizer. ## Huggingface tokenizer Compatible with https://github.com/huggingface/tokenizers/. +Optional C++ chat template rendering is available with +`TOKENIZERS_BUILD_CHAT_TEMPLATE=ON`. See the +[chat template example](examples/chat_template_tool/README.md) for loading HF +templates and rendering prompts separately from tokenization and generation. + ## Llama2.c tokenizer Adapted from https://github.com/karpathy/llama2.c. diff --git a/examples/chat_template_tool/CMakeLists.txt b/examples/chat_template_tool/CMakeLists.txt new file mode 100644 index 00000000..c9d7151e --- /dev/null +++ b/examples/chat_template_tool/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(chat_template_tool main.cpp) +target_link_libraries(chat_template_tool PRIVATE tokenizers::chat_template) diff --git a/examples/chat_template_tool/README.md b/examples/chat_template_tool/README.md new file mode 100644 index 00000000..0fbbbcb1 --- /dev/null +++ b/examples/chat_template_tool/README.md @@ -0,0 +1,61 @@ +# HF chat template rendering + +Enable `TOKENIZERS_BUILD_CHAT_TEMPLATE` and `TOKENIZERS_BUILD_TOOLS`, then build +`chat_template_tool`. This optional target uses the pinned Minja Jinja renderer; +the default tokenizer build does not compile it. + +```bash +cmake -S . -B build -DTOKENIZERS_BUILD_CHAT_TEMPLATE=ON -DTOKENIZERS_BUILD_TOOLS=ON +cmake --build build --target chat_template_tool +build/examples/chat_template_tool/chat_template_tool model_dir messages.json kwargs.json +``` + +`messages.json` uses HF's role/content conversation structure, for example: + +```json +[{"role": "user", "content": "Give me a short introduction to large language models."}] +``` + +`kwargs.json` can contain `{"enable_thinking": false}` or other variables used by +the model template. A final `0` or `1` argument controls `add_generation_prompt` +(default `1`). Output is the rendered prompt, with no additional newline. + +C++ runners can link `tokenizers::chat_template` and use the same API: + +```cpp +tokenizers::HFChatTemplate renderer; +if (renderer.load(model_directory) != tokenizers::Error::Ok) { + // Report a missing/unsupported template or choose an explicit app fallback. +} +auto prompt = renderer.apply(messages, true, {{"enable_thinking", false}}); +if (prompt.ok()) { + auto ids = tokenizer.encode(*prompt, 0, 0); +} +``` + +Rendering is separate from tokenization and generation. Encode without adding +BOS/EOS again because the model's template supplies the required special tokens. +`enable_thinking` changes only templates that reference that variable. + +The loader reads `chat_template.jinja`, legacy `tokenizer_config.json` templates +or legacy processor `chat_template.json`, plus string/object special-token +metadata. Standalone Jinja overrides legacy templates. Named templates can be +selected with `load(directory, "tool_use")`; automatic tool-template selection is +not performed. Named standalone files live in `additional_chat_templates`. + +Missing artifacts return `LoadFailure`; malformed/unsupported templates and +rendering failures return `ParseFailure`. Failed reloads preserve the old template. +Each artifact is limited to 1 MiB. Load before sharing a renderer; concurrent +`apply` calls have independent context, but concurrent reloads are not supported. +Use trusted model templates: Minja implements a subset of Jinja for chat models, +not every Python/Jinja feature. This first library/example change does not alter +existing ExecuTorch app prompts or add a Kotlin/JNI binding for this new API. +The extra `capitalize` filter covers ASCII role names; Unicode case expansion is +unsupported and returns a render error. Multimodal messages must use the content +block structure expected by their processor template. + +Fixtures pin Qwen3 and SmolVLM templates and contain exact reference output from +Transformers, covering Unicode, multiple turns, image content, generation prompts +and thinking enabled/disabled/default. Source revisions and hashes are in +`test/resources/chat_templates/manifest.json`. Fixture templates are Apache-2.0; +Minja is MIT. The normal project tests include the renderer when the option is on. diff --git a/examples/chat_template_tool/main.cpp b/examples/chat_template_tool/main.cpp new file mode 100644 index 00000000..178167a5 --- /dev/null +++ b/examples/chat_template_tool/main.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include +#include +#include + +int main(int argc, char** argv) { + if (argc < 3 || argc > 5) { + std::cerr + << "Usage: chat_template_tool MODEL_DIR MESSAGES_JSON [KWARGS_JSON] [0|1 add_generation_prompt]\n"; + return 1; + } + try { + std::ifstream messages_file(argv[2]); + const auto messages = nlohmann::ordered_json::parse(messages_file); + auto kwargs = nlohmann::ordered_json::object(); + if (argc >= 4) { + std::ifstream kwargs_file(argv[3]); + kwargs = nlohmann::ordered_json::parse(kwargs_file); + } + if (argc == 5 && std::string(argv[4]) != "0" && + std::string(argv[4]) != "1") { + throw std::runtime_error("Generation prompt must be 0 or 1"); + } + tokenizers::HFChatTemplate renderer; + if (renderer.load(argv[1]) != tokenizers::Error::Ok) { + throw std::runtime_error("Cannot load HF chat template"); + } + const auto prompt = renderer.apply( + messages, argc < 5 || std::string(argv[4]) == "1", kwargs); + if (!prompt.ok()) { + throw std::runtime_error("Cannot render HF chat template"); + } + std::cout << *prompt; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/include/pytorch/tokenizers/chat_template.h b/include/pytorch/tokenizers/chat_template.h new file mode 100644 index 00000000..cd233495 --- /dev/null +++ b/include/pytorch/tokenizers/chat_template.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#pragma once + +#include +#include +#include +#include + +namespace tokenizers { + +/** Render HF chat templates without loading model weights or generating text. + */ +class HFChatTemplate { + public: + HFChatTemplate(); + ~HFChatTemplate(); + HFChatTemplate(const HFChatTemplate&) = delete; + HFChatTemplate& operator=(const HFChatTemplate&) = delete; + + /** + * Load a model directory, tokenizer_config.json, chat_template.json or Jinja. + * Standalone Jinja takes precedence over the config's legacy template. + * Select a named template explicitly; the default name is "default". + * A failed reload preserves the previously loaded template. + */ + Error load(const std::string& path, const std::string& name = "default"); + + /** + * Equivalent to rendering apply_chat_template(..., tokenize=False). + * kwargs can include enable_thinking, tools and other model-specific values. + * messages and add_generation_prompt are supplied by the dedicated arguments. + * Encode the returned text with BOS/EOS counts zero to avoid duplicate + * tokens. + */ + Result apply( + const nlohmann::ordered_json& messages, + bool add_generation_prompt = true, + const nlohmann::ordered_json& kwargs = + nlohmann::ordered_json::object()) const; + + bool is_loaded() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace tokenizers diff --git a/src/chat_template.cpp b/src/chat_template.cpp new file mode 100644 index 00000000..3ae43156 --- /dev/null +++ b/src/chat_template.cpp @@ -0,0 +1,248 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace tokenizers { +namespace { +using Json = nlohmann::ordered_json; +namespace fs = std::filesystem; +constexpr std::uintmax_t kMaxArtifactBytes = 1024 * 1024; + +std::string read(const fs::path& path) { + if (fs::file_size(path) > kMaxArtifactBytes) { + throw std::runtime_error("Chat template artifact exceeds 1 MiB"); + } + std::ifstream stream(path, std::ios::binary); + if (!stream) { + throw std::runtime_error("Cannot open chat template artifact"); + } + std::string result( + (std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + if (stream.bad()) { + throw std::runtime_error("Cannot read chat template artifact"); + } + return result; +} + +std::string legacy_template(const Json& value, const std::string& name) { + if (value.is_string()) { + return name == "default" ? value.get() : ""; + } + if (value.is_object() && value.contains(name) && value[name].is_string()) { + return value[name].get(); + } + if (value.is_array()) { + for (const auto& entry : value) { + if (entry.is_object() && entry.value("name", "") == name && + entry.contains("template") && entry["template"].is_string()) { + return entry["template"].get(); + } + } + } + return ""; +} + +void special_tokens(const Json& config, Json& context) { + for (auto it = config.begin(); it != config.end(); ++it) { + const auto& key = it.key(); + if (key.size() < 6 || key.compare(key.size() - 6, 6, "_token") != 0) { + continue; + } + if (it.value().is_string()) { + context[key] = it.value(); + } else if ( + it.value().is_object() && it.value().contains("content") && + it.value()["content"].is_string()) { + context[key] = it.value()["content"]; + } + } +} +} // namespace + +struct HFChatTemplate::Impl { + std::shared_ptr node; + Json tokens = Json::object(); +}; + +HFChatTemplate::HFChatTemplate() = default; +HFChatTemplate::~HFChatTemplate() = default; +bool HFChatTemplate::is_loaded() const { + return impl_ != nullptr; +} + +Error HFChatTemplate::load(const std::string& path, const std::string& name) { + // Named templates are file names under additional_chat_templates. + if (name.empty() || + !std::all_of(name.begin(), name.end(), [](unsigned char ch) { + return std::isalnum(ch) || ch == '_' || ch == '-'; + })) { + return Error::ParseFailure; + } + std::error_code error; + const fs::path artifact(path); + if (!fs::exists(artifact, error) || error) { + return Error::LoadFailure; + } + try { + const bool directory = fs::is_directory(artifact); + const fs::path root = directory ? artifact : artifact.parent_path(); + auto next = std::make_unique(); + std::string source; + const auto config_path = root / "tokenizer_config.json"; + if (fs::exists(config_path)) { + const auto config = Json::parse(read(config_path)); + if (!config.is_object()) { + return Error::ParseFailure; + } + special_tokens(config, next->tokens); + if (config.contains("chat_template")) { + source = legacy_template(config["chat_template"], name); + } + } + const auto tokens_path = root / "special_tokens_map.json"; + if (fs::exists(tokens_path)) { + const auto tokens = Json::parse(read(tokens_path)); + if (!tokens.is_object()) { + return Error::ParseFailure; + } + special_tokens(tokens, next->tokens); + } + const auto processor_path = root / "chat_template.json"; + if (source.empty() && fs::exists(processor_path)) { + const auto processor = Json::parse(read(processor_path)); + if (processor.contains("chat_template")) { + source = legacy_template(processor["chat_template"], name); + } + } + fs::path standalone = name == "default" + ? root / "chat_template.jinja" + : root / "additional_chat_templates" / (name + ".jinja"); + if (!directory && artifact.extension() == ".jinja") { + standalone = artifact; + } + if (fs::exists(standalone)) { + source = read(standalone); + } + if (source.empty()) { + return Error::LoadFailure; + } + // Jinja normalizes template source newlines, including inside literals. + // Preserve message content: only the template source is normalized. + std::string normalized; + normalized.reserve(source.size()); + for (size_t i = 0; i < source.size(); ++i) { + if (source[i] == '\r') { + normalized += '\n'; + if (i + 1 < source.size() && source[i + 1] == '\n') + ++i; + } else { + normalized += source[i]; + } + } + minja::Options options{}; + options.trim_blocks = true; + options.lstrip_blocks = true; + next->node = minja::Parser::parse(normalized, options); + impl_ = std::move(next); + return Error::Ok; + } catch (const std::exception&) { + return Error::ParseFailure; + } +} + +Result HFChatTemplate::apply( + const Json& messages, + bool add_generation_prompt, + const Json& kwargs) const { + if (!impl_) { + return Error::Uninitialized; + } + if (!messages.is_array() || messages.empty() || !kwargs.is_object() || + kwargs.contains("messages") || kwargs.contains("add_generation_prompt")) { + return Error::ParseFailure; + } + for (const auto& message : messages) { + if (!message.is_object() || !message.contains("role") || + !message["role"].is_string() || !message.contains("content")) { + return Error::ParseFailure; + } + } + try { + Json context = impl_->tokens; + context.update(kwargs); + context["messages"] = messages; + context["add_generation_prompt"] = add_generation_prompt; + // Per-call context avoids leaking mutable Jinja namespace values across + // turns or between concurrent callers. + auto environment = minja::Context::make(minja::Value(context)); + environment->set( + "capitalize", + minja::simple_function( + "capitalize", + {"text"}, + [](const std::shared_ptr&, + minja::Value& args) -> minja::Value { + auto text = args.at("text").get(); + // HF role names are ASCII. Do not silently approximate Python's + // Unicode title/lower mappings (which can expand code points). + for (size_t i = 0; i < text.size(); ++i) { + const auto ch = static_cast(text[i]); + if (ch >= 128) { + throw std::runtime_error("Unicode capitalize is unsupported"); + } + if (i == 0 && ch >= 'a' && ch <= 'z') + text[i] -= 'a' - 'A'; + if (i != 0 && ch >= 'A' && ch <= 'Z') + text[i] += 'a' - 'A'; + } + return minja::Value(text); + })); + environment->set( + "strftime_now", + minja::simple_function( + "strftime_now", + {"format"}, + [](const std::shared_ptr&, + minja::Value& args) -> minja::Value { + const auto format = args.at("format").get(); + if (format.empty()) + return minja::Value(""); + const auto now = std::time(nullptr); + std::tm local{}; +#ifdef _WIN32 + if (localtime_s(&local, &now) != 0) + throw std::runtime_error("Cannot read local time"); +#else + if (localtime_r(&now, &local) == nullptr) + throw std::runtime_error("Cannot read local time"); +#endif + std::vector buffer(128); + while (buffer.size() <= 4096) { + const auto count = std::strftime( + buffer.data(), buffer.size(), format.c_str(), &local); + if (count) + return minja::Value(std::string(buffer.data(), count)); + buffer.resize(buffer.size() * 2); + } + throw std::runtime_error("Date format exceeds output limit"); + })); + return impl_->node->render(environment); + } catch (const std::exception&) { + return Error::ParseFailure; + } +} +} // namespace tokenizers diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b9e0a034..c00e35da 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,6 +32,9 @@ set(gtest_force_shared_crt FetchContent_MakeAvailable(googletest) file(GLOB test_source_files test_*.cpp) +if(NOT TOKENIZERS_BUILD_CHAT_TEMPLATE) + list(FILTER test_source_files EXCLUDE REGEX "test_chat_template\\.cpp$") +endif() set(test_env "RESOURCES_PATH=${CMAKE_CURRENT_SOURCE_DIR}/resources") enable_testing() @@ -43,6 +46,9 @@ foreach(test_source_file ${test_source_files}) ${test_name} PRIVATE GTEST_INCLUDE_PATH ${TOKENIZERS_INCLUDE_DIRS} ) target_link_libraries(${test_name} PUBLIC gtest_main GTest::gmock tokenizers) + if(test_name STREQUAL "test_chat_template") + target_link_libraries(${test_name} PRIVATE tokenizers::chat_template) + endif() add_test(${test_name} "${test_name}") set_tests_properties(${test_name} PROPERTIES ENVIRONMENT ${test_env}) endforeach() diff --git a/test/resources/chat_templates/LICENSE b/test/resources/chat_templates/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/test/resources/chat_templates/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/resources/chat_templates/cases.json b/test/resources/chat_templates/cases.json new file mode 100644 index 00000000..d6100608 --- /dev/null +++ b/test/resources/chat_templates/cases.json @@ -0,0 +1,744 @@ +[ + { + "model": "qwen3", + "messages": [ + { + "role": "user", + "content": "Hello Café 世界 👋" + } + ], + "add_generation_prompt": false, + "kwargs": {}, + "expected": "<|im_start|>user\nHello Café 世界 👋<|im_end|>\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "user", + "content": "Hello Café 世界 👋" + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>user\nHello Café 世界 👋<|im_end|>\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "user", + "content": "Hello Café 世界 👋" + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>user\nHello Café 世界 👋<|im_end|>\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "user", + "content": "Hello Café 世界 👋" + } + ], + "add_generation_prompt": true, + "kwargs": {}, + "expected": "<|im_start|>user\nHello Café 世界 👋<|im_end|>\n<|im_start|>assistant\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "user", + "content": "Hello Café 世界 👋" + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>user\nHello Café 世界 👋<|im_end|>\n<|im_start|>assistant\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "user", + "content": "Hello Café 世界 👋" + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>user\nHello Café 世界 👋<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "system", + "content": "Be brief." + }, + { + "role": "user", + "content": "What is 2+2?" + }, + { + "role": "assistant", + "content": "4." + }, + { + "role": "user", + "content": "Why?" + } + ], + "add_generation_prompt": false, + "kwargs": {}, + "expected": "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n4.<|im_end|>\n<|im_start|>user\nWhy?<|im_end|>\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "system", + "content": "Be brief." + }, + { + "role": "user", + "content": "What is 2+2?" + }, + { + "role": "assistant", + "content": "4." + }, + { + "role": "user", + "content": "Why?" + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n4.<|im_end|>\n<|im_start|>user\nWhy?<|im_end|>\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "system", + "content": "Be brief." + }, + { + "role": "user", + "content": "What is 2+2?" + }, + { + "role": "assistant", + "content": "4." + }, + { + "role": "user", + "content": "Why?" + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n4.<|im_end|>\n<|im_start|>user\nWhy?<|im_end|>\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "system", + "content": "Be brief." + }, + { + "role": "user", + "content": "What is 2+2?" + }, + { + "role": "assistant", + "content": "4." + }, + { + "role": "user", + "content": "Why?" + } + ], + "add_generation_prompt": true, + "kwargs": {}, + "expected": "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n4.<|im_end|>\n<|im_start|>user\nWhy?<|im_end|>\n<|im_start|>assistant\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "system", + "content": "Be brief." + }, + { + "role": "user", + "content": "What is 2+2?" + }, + { + "role": "assistant", + "content": "4." + }, + { + "role": "user", + "content": "Why?" + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n4.<|im_end|>\n<|im_start|>user\nWhy?<|im_end|>\n<|im_start|>assistant\n" + }, + { + "model": "qwen3", + "messages": [ + { + "role": "system", + "content": "Be brief." + }, + { + "role": "user", + "content": "What is 2+2?" + }, + { + "role": "assistant", + "content": "4." + }, + { + "role": "user", + "content": "Why?" + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n4.<|im_end|>\n<|im_start|>user\nWhy?<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello Café 世界 👋" + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": {}, + "expected": "<|im_start|>User: Hello Café 世界 👋\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello Café 世界 👋" + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>User: Hello Café 世界 👋\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello Café 世界 👋" + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>User: Hello Café 世界 👋\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello Café 世界 👋" + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": {}, + "expected": "<|im_start|>User: Hello Café 世界 👋\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello Café 世界 👋" + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>User: Hello Café 世界 👋\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello Café 世界 👋" + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>User: Hello Café 世界 👋\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Be brief." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Why?" + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": {}, + "expected": "<|im_start|>System: Be brief.\nUser: What is 2+2?\nAssistant: 4.\nUser: Why?\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Be brief." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Why?" + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>System: Be brief.\nUser: What is 2+2?\nAssistant: 4.\nUser: Why?\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Be brief." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Why?" + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>System: Be brief.\nUser: What is 2+2?\nAssistant: 4.\nUser: Why?\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Be brief." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Why?" + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": {}, + "expected": "<|im_start|>System: Be brief.\nUser: What is 2+2?\nAssistant: 4.\nUser: Why?\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Be brief." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Why?" + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>System: Be brief.\nUser: What is 2+2?\nAssistant: 4.\nUser: Why?\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Be brief." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Why?" + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>System: Be brief.\nUser: What is 2+2?\nAssistant: 4.\nUser: Why?\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": "Describe this photo." + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": {}, + "expected": "<|im_start|>User:Describe this photo.\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": "Describe this photo." + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>User:Describe this photo.\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": "Describe this photo." + } + ] + } + ], + "add_generation_prompt": false, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>User:Describe this photo.\n" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": "Describe this photo." + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": {}, + "expected": "<|im_start|>User:Describe this photo.\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": "Describe this photo." + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": true + }, + "expected": "<|im_start|>User:Describe this photo.\nAssistant:" + }, + { + "model": "smolvlm", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": "Describe this photo." + } + ] + } + ], + "add_generation_prompt": true, + "kwargs": { + "enable_thinking": false + }, + "expected": "<|im_start|>User:Describe this photo.\nAssistant:" + } +] diff --git a/test/resources/chat_templates/manifest.json b/test/resources/chat_templates/manifest.json new file mode 100644 index 00000000..07e37d3d --- /dev/null +++ b/test/resources/chat_templates/manifest.json @@ -0,0 +1,23 @@ +{ + "transformers_version": "5.18.0", + "fixtures": [ + { + "model": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "template_source": "tokenizer_config.json", + "url": "https://huggingface.co/Qwen/Qwen3-0.6B/blob/c1899de289a04d12100db370d81485cdf75e47ca/tokenizer_config.json", + "license": "Apache-2.0", + "template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", + "fixture_template_sha256": "e132ae041e1217b5e1114eb9dc292484a7f478df945d72fa49ba01b88d8ec01a" + }, + { + "model": "HuggingFaceTB/SmolVLM-256M-Instruct", + "revision": "7e3e67edbbed1bf9888184d9df282b700a323964", + "template_source": "chat_template.json", + "url": "https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Instruct/blob/7e3e67edbbed1bf9888184d9df282b700a323964/chat_template.json", + "license": "Apache-2.0", + "template_sha256": "830005085ddd94cec605b6e21b885b28ffad6e3f82ee6a212c523533e8f04f01", + "fixture_template_sha256": "9e6a479b8d92b445c22f8c510a3df862ba14f4c18c7a02a6365e380d73f45ece" + } + ] +} diff --git a/test/resources/chat_templates/qwen3/chat_template.jinja b/test/resources/chat_templates/qwen3/chat_template.jinja new file mode 100644 index 00000000..9769e5cf --- /dev/null +++ b/test/resources/chat_templates/qwen3/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} diff --git a/test/resources/chat_templates/qwen3/tokenizer_config.json b/test/resources/chat_templates/qwen3/tokenizer_config.json new file mode 100644 index 00000000..e131555f --- /dev/null +++ b/test/resources/chat_templates/qwen3/tokenizer_config.json @@ -0,0 +1,4 @@ +{ + "eos_token": "<|im_end|>", + "pad_token": "<|endoftext|>" +} diff --git a/test/resources/chat_templates/smolvlm/chat_template.jinja b/test/resources/chat_templates/smolvlm/chat_template.jinja new file mode 100644 index 00000000..969fd255 --- /dev/null +++ b/test/resources/chat_templates/smolvlm/chat_template.jinja @@ -0,0 +1,2 @@ +<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %} +{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %} diff --git a/test/resources/chat_templates/smolvlm/tokenizer_config.json b/test/resources/chat_templates/smolvlm/tokenizer_config.json new file mode 100644 index 00000000..6cfeece1 --- /dev/null +++ b/test/resources/chat_templates/smolvlm/tokenizer_config.json @@ -0,0 +1,6 @@ +{ + "bos_token": "<|im_start|>", + "eos_token": "", + "pad_token": "<|im_end|>", + "unk_token": "<|endoftext|>" +} diff --git a/test/test_chat_template.cpp b/test/test_chat_template.cpp new file mode 100644 index 00000000..7342ad3c --- /dev/null +++ b/test/test_chat_template.cpp @@ -0,0 +1,262 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace tokenizers { +namespace { +using Json = nlohmann::ordered_json; +namespace fs = std::filesystem; + +struct Directory { + fs::path path; + Directory() { + static std::atomic next{0}; + path = fs::temp_directory_path() / + ("hf-chat-template-" + std::to_string(next++) + "-" + + std::to_string( + std::chrono::steady_clock::now().time_since_epoch().count())); + fs::create_directories(path); + } + ~Directory() { + std::error_code error; + fs::remove_all(path, error); + } + void write(const std::string& name, const std::string& content) { + fs::create_directories((path / name).parent_path()); + std::ofstream(path / name, std::ios::binary) << content; + } +}; + +const Json kMessages = + Json::array({{{"role", "user"}, {"content", "Café 世界 👋"}}}); + +TEST(HFChatTemplateTest, UninitializedAndMissingArtifact) { + HFChatTemplate renderer; + EXPECT_EQ(renderer.apply(kMessages).error(), Error::Uninitialized); + EXPECT_EQ(renderer.load("/missing/chat_template.jinja"), Error::LoadFailure); + EXPECT_FALSE(renderer.is_loaded()); +} + +TEST(HFChatTemplateTest, StandaloneTakesPrecedenceAndUsesSpecialTokenContent) { + Directory directory; + directory.write( + "tokenizer_config.json", + R"({"chat_template":"wrong","bos_token":{"content":""},"eos_token":""})"); + directory.write( + "chat_template.jinja", + "{{ bos_token }}{{ messages[0]['content'] }}{{ eos_token }}{% if add_generation_prompt %}Assistant:{% endif %}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + auto rendered = renderer.apply(kMessages); + ASSERT_TRUE(rendered.ok()); + EXPECT_EQ(*rendered, "Café 世界 👋Assistant:"); + EXPECT_EQ(*renderer.apply(kMessages, false), "Café 世界 👋"); +} + +TEST(HFChatTemplateTest, LegacyNamedAndStandaloneNamedTemplates) { + Directory directory; + directory.write( + "tokenizer_config.json", + R"({"chat_template":[{"name":"default","template":"DEFAULT"},{"name":"tool_use","template":"LEGACY TOOL"}]})"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "DEFAULT"); + ASSERT_EQ(renderer.load(directory.path.string(), "tool_use"), Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "LEGACY TOOL"); + directory.write( + "additional_chat_templates/tool_use.jinja", "{{ tools[0]['name'] }}"); + ASSERT_EQ(renderer.load(directory.path.string(), "tool_use"), Error::Ok); + EXPECT_EQ( + *renderer.apply( + kMessages, true, {{"tools", Json::array({{{"name", "camera"}}})}}), + "camera"); + EXPECT_EQ( + renderer.load(directory.path.string(), "../escape"), Error::ParseFailure); +} + +TEST(HFChatTemplateTest, LegacyStringAndJsonObjectTemplate) { + Directory directory; + directory.write( + "tokenizer_config.json", + R"({"chat_template":"{{ messages[0]['role'] }}"})"); + HFChatTemplate renderer; + ASSERT_EQ( + renderer.load((directory.path / "tokenizer_config.json").string()), + Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "user"); + directory.write( + "tokenizer_config.json", R"({"chat_template":{"default":"OBJECT"}})"); + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "OBJECT"); +} + +TEST(HFChatTemplateTest, FailedReloadPreservesPreviousTemplate) { + Directory directory; + directory.write("chat_template.jinja", "VALID"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + directory.write("chat_template.jinja", "{% if true %}"); + EXPECT_EQ(renderer.load(directory.path.string()), Error::ParseFailure); + EXPECT_TRUE(renderer.is_loaded()); + EXPECT_EQ(*renderer.apply(kMessages), "VALID"); + directory.write("tokenizer_config.json", "{"); + EXPECT_EQ(renderer.load(directory.path.string()), Error::ParseFailure); +} + +TEST(HFChatTemplateTest, InvalidMessagesAndReservedKwargsFail) { + Directory directory; + directory.write("chat_template.jinja", "VALID"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ(renderer.apply(Json::array()).error(), Error::ParseFailure); + EXPECT_EQ(renderer.apply(Json::object()).error(), Error::ParseFailure); + EXPECT_EQ( + renderer.apply(Json::array({{{"content", "missing role"}}})).error(), + Error::ParseFailure); + EXPECT_EQ( + renderer.apply(kMessages, true, {{"messages", "override"}}).error(), + Error::ParseFailure); + EXPECT_EQ( + renderer.apply(kMessages, true, {{"add_generation_prompt", false}}) + .error(), + Error::ParseFailure); +} + +TEST(HFChatTemplateTest, RuntimeExceptionIsAnError) { + Directory directory; + directory.write( + "chat_template.jinja", "{{ raise_exception('Invalid conversation') }}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ(renderer.apply(kMessages).error(), Error::ParseFailure); +} + +TEST(HFChatTemplateTest, LegacyProcessorTemplateIsLoaded) { + Directory directory; + directory.write( + "chat_template.json", + R"({"chat_template":"{{ messages[0]['content'] }}"})"); + HFChatTemplate renderer; + ASSERT_EQ( + renderer.load((directory.path / "chat_template.json").string()), + Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "Café 世界 👋"); +} + +TEST(HFChatTemplateTest, CurrentDateHelperIsAvailable) { + Directory directory; + directory.write("chat_template.jinja", "{{ strftime_now('%Y') }}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + const auto now = std::time(nullptr); + std::tm local{}; +#ifdef _WIN32 + ASSERT_EQ(localtime_s(&local, &now), 0); +#else + ASSERT_NE(localtime_r(&now, &local), nullptr); +#endif + EXPECT_EQ(*renderer.apply(kMessages), std::to_string(1900 + local.tm_year)); +} + +TEST(HFChatTemplateTest, ThinkingAndMutableContextArePerCall) { + Directory directory; + directory.write( + "chat_template.jinja", + "{% set ns = namespace(value=0) %}{% set ns.value = ns.value + 1 %}{{ ns.value }}{% if enable_thinking is defined and not enable_thinking %}OFF{% else %}ON{% endif %}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "1ON"); + EXPECT_EQ( + *renderer.apply(kMessages, true, {{"enable_thinking", false}}), "1OFF"); + EXPECT_EQ( + *renderer.apply(kMessages, true, {{"enable_thinking", true}}), "1ON"); +} + +TEST( + HFChatTemplateTest, + CapitalizeHandlesAsciiRolesAndRejectsUnicodeApproximation) { + Directory directory; + directory.write( + "chat_template.jinja", "{{ messages[0]['role'] | capitalize }}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ(*renderer.apply(kMessages), "User"); + EXPECT_EQ( + *renderer.apply(Json::array({{{"role", "aSSISTANT"}, {"content", ""}}})), + "Assistant"); + EXPECT_EQ( + renderer.apply(Json::array({{{"role", "ß"}, {"content", ""}}})).error(), + Error::ParseFailure); +} + +TEST(HFChatTemplateTest, ConcurrentRenderingKeepsMessagesSeparate) { + Directory directory; + directory.write("chat_template.jinja", "{{ messages[0]['content'] }}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + std::atomic success{true}; + std::vector threads; + for (unsigned i = 0; i < 3; ++i) { + threads.emplace_back([&, i] { + for (unsigned n = 0; n < 20; ++n) { + const auto text = std::to_string(i) + ":" + std::to_string(n); + const auto rendered = renderer.apply( + Json::array({{{"role", "user"}, {"content", text}}})); + if (!rendered.ok() || *rendered != text) + success = false; + } + }); + } + for (auto& thread : threads) + thread.join(); + EXPECT_TRUE(success); +} + +TEST(HFChatTemplateTest, TemplateNewlinesMatchJinjaWithoutChangingMessages) { + Directory directory; + directory.write( + "chat_template.jinja", + "{{ 'first\r\nsecond\rthird' }}{{ messages[0]['content'] }}"); + HFChatTemplate renderer; + ASSERT_EQ(renderer.load(directory.path.string()), Error::Ok); + EXPECT_EQ( + *renderer.apply(Json::array({{{"role", "user"}, {"content", "\r\n"}}})), + "first\nsecond\nthird\r\n"); +} + +TEST(HFChatTemplateTest, ExactTransformersParityForPinnedRealModels) { + const auto root = fs::path(std::getenv("RESOURCES_PATH")) / "chat_templates"; + std::ifstream input(root / "cases.json"); + const auto cases = Json::parse(input); + ASSERT_GE(cases.size(), 30); + for (size_t i = 0; i < cases.size(); ++i) { + SCOPED_TRACE("HF parity case " + std::to_string(i)); + const auto& entry = cases[i]; + HFChatTemplate renderer; + ASSERT_EQ( + renderer.load((root / entry["model"].get()).string()), + Error::Ok); + const auto actual = renderer.apply( + entry["messages"], + entry["add_generation_prompt"].get(), + entry["kwargs"]); + ASSERT_TRUE(actual.ok()); + EXPECT_EQ(*actual, entry["expected"].get()); + } +} +} // namespace +} // namespace tokenizers diff --git a/third-party/minja b/third-party/minja new file mode 160000 index 00000000..021c2293 --- /dev/null +++ b/third-party/minja @@ -0,0 +1 @@ +Subproject commit 021c2293c187789ef13d56c6cfd89c9b134fd80f