CJK text wrapping tests for imgui#8890

imgui_test_engine/imgui_te_utils.cpp

@@ -594,6 +594,21 @@ int ImStrBase64Encode(const unsigned char* src, char* dst, int length)
     return encoded_len;
 }
 
+// Collect Utf8 character offsets
+size_t ImStrUtf8CollectOffsets(const char* in_text, const char* in_text_end, size_t* out_offsets, size_t out_offsets_size) {
+    const char* s = in_text;
+    size_t total_chars = 0;
+    out_offsets[total_chars++] = 0;  // first offset always 0
+    while (s < in_text_end && total_chars < out_offsets_size) {
+        unsigned int unused_c;
+        s += ImTextCharFromUtf8(&unused_c, s, in_text_end);
+        if (s < in_text_end) {
+            out_offsets[total_chars++] = s - in_text;
+        }
+    }
+    return total_chars;
+}
+
 //-----------------------------------------------------------------------------
 // Parsing Helpers
 //-----------------------------------------------------------------------------

imgui_test_engine/imgui_te_utils.h

@@ -50,7 +50,7 @@ void        ImStrReplace(Str* s, const char* find, const char* repl);
 const char* ImStrchrRangeWithEscaping(const char* str, const char* str_end, char find_c);
 void        ImStrXmlEscape(Str* s);
 int         ImStrBase64Encode(const unsigned char* src, char* dst, int length);
-
+size_t      ImStrUtf8CollectOffsets(const char* in_text, const char* in_text_end, size_t* out_offsets, size_t out_offsets_size);
 //-----------------------------------------------------------------------------
 // Parsing Helpers
 //-----------------------------------------------------------------------------

imgui_test_suite/assets/fonts/LICENSE_SourceHanSansCN.txt

@@ -0,0 +1,96 @@
+Copyright 2014-2025 Adobe (http://www.adobe.com/), with Reserved Font
+Name 'Source'. Source is a trademark of Adobe in the United States
+and/or other countries.
+
+This Font Software is licensed under the SIL Open Font License,
+Version 1.1.
+
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font
+creation efforts of academic and linguistic communities, and to
+provide a free and open framework in which fonts may be shared and
+improved in partnership with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply to
+any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software
+components as distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to,
+deleting, or substituting -- in part or in whole -- any of the
+components of the Original Version, by changing formats or by porting
+the Font Software to a new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed,
+modify, redistribute, and sell modified and unmodified copies of the
+Font Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components, in
+Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the
+corresponding Copyright Holder. This restriction only applies to the
+primary font name as presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created using
+the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

imgui_test_suite/imgui_test_suite.cpp

@@ -308,6 +308,7 @@ static void TestSuite_LoadFonts(float dpi_scale)
     {
         io.Fonts->AddFontFromFileTTF(Str64f("%s/%s", base_font_dir.c_str(), "NotoSans-Regular.ttf").c_str(), IM_ROUND(16.0f * dpi_scale));
         io.Fonts->AddFontFromFileTTF(Str64f("%s/%s", base_font_dir.c_str(), "Roboto-Medium.ttf").c_str(), IM_ROUND(16.0f * dpi_scale));
+        io.Fonts->AddFontFromFileTTF(Str64f("%s/%s", base_font_dir.c_str(), "SourceHanSansCN-VF-subset.ttf").c_str(), IM_ROUND(16.0f * dpi_scale));
         //io.Fonts->AddFontFromFileTTF(Str64f("%s/%s", base_font_dir.c_str(), "RobotoMono-Regular.ttf").c_str(), 16.0f * dpi_scale, &cfg);
         //io.Fonts->AddFontFromFileTTF(Str64f("%s/%s", base_font_dir.c_str(), "Cousine-Regular.ttf").c_str(), 15.0f * dpi_scale);
         //io.Fonts->AddFontFromFileTTF(Str64f("%s/%s", base_font_dir.c_str(), "DroidSans.ttf").c_str(), 16.0f * dpi_scale);

imgui_test_suite/imgui_test_suite.h

@@ -3,7 +3,8 @@
 
 #pragma once
 
-#define TEST_SUITE_ALT_FONT_NAME   "Roboto-Medium.ttf"      // Prefix
+#define TEST_SUITE_ALT_FONT_NAME       "Roboto-Medium.ttf"      // Prefix
+#define TEST_SUITE_ALT_FONT_NAME_CJK   "SourceHanSansCN-VF-subset.ttf"      // Prefix
 
 // Tests Registration Functions
 struct ImGuiTestEngine;

imgui_test_suite/imgui_tests_widgets.cpp

@@ -10,6 +10,7 @@
 #include "imgui_test_engine/imgui_te_context.h"
 #include "imgui_test_engine/imgui_te_utils.h"       // InputText() with Str
 #include "imgui_test_engine/thirdparty/Str/Str.h"
+#include "imgui_test_suite.h"                       // TEST_SUITE_ALT_FONT_NAME_CJK
 
 // Warnings
 #ifdef _MSC_VER
@@ -3584,7 +3585,7 @@ void RegisterTests_Widgets(ImGuiTestEngine* e)
         ImGui::LogToClipboard();
 #ifdef __GNUC__
 #pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wformat-overflow"  // warning: �%s� directive argument is null
+#pragma GCC diagnostic ignored "-Wformat-overflow" // warning: ‘%s’ directive argument is null
 #endif
         ImGui::Text("%s", (const char*)NULL);
         ImGui::Text("%.*s", 3, (const char*)NULL);
@@ -3677,6 +3678,157 @@ void RegisterTests_Widgets(ImGuiTestEngine* e)
 #endif
     };
 
+#if (IMGUI_VERSION_NUM >= 19197 || defined(IMGUI_HAS_TEXTURES)) && defined(IMGUI_TEST_ENGINE_ENABLE_STD_FUNCTION)
+
+    // # Legacy test
+    t = IM_REGISTER_TEST(e, "widgets", "widgets_text_wrapped_cjk_legacy_headless");
+    t->GuiFunc = [](ImGuiTestContext* ctx)
+    {
+        ImGui::Begin("Test Window", NULL, ImGuiWindowFlags_NoSavedSettings | ImGuiWindowFlags_NoTitleBar);
+        const char* s1 = "abcde..";
+        const char* s2 = "abcde.. That";
+        float local_off_x = ImGui::GetCursorPos().x;
+        float wrap_width = ImGui::CalcTextSize(s1).x;
+        // Visualize output here, no actual test
+        ImGui::PushTextWrapPos(wrap_width + local_off_x);
+        ImGui::TextUnformatted(s1);
+        ImGui::Spacing();
+        ImGui::TextUnformatted(s2); // Though the '.' can fit in the first line, it's now at the second line.
+        ImGui::PopTextWrapPos();
+
+        // Text wrapping with leading \n
+        ImGui::Separator();
+        ImGui::TextWrapped("\nHello");
+        IM_CHECK_EQ(ImGui::GetItemRectSize(), ImVec2(ImGui::CalcTextSize("Hello").x, ImGui::GetFontSize() * 2));
+
+        const char* s2_w = ImGui::GetFont()->CalcWordWrapPositionCJK(ImGui::GetFontSize(), s2, s2 + strlen(s1), wrap_width);
+        IM_CHECK_EQ(s2_w, s2 + strlen(s1));
+        ImGui::End();
+    };
+
+    /* this test case is not really needed, it's covered by the rest cases
+    // ## Test pure CJK(zh-cn) characters
+    // No special case in this test, the wrap can happen at at every char
+    t = IM_REGISTER_TEST(e, "widgets", "widgets_text_wrapped_cjk_zh_cn_normal_chars_headless");
+    t->GuiFunc = [](ImGuiTestContext* ctx)
+    {
+        ImGui::Begin("Test Window", NULL, ImGuiWindowFlags_NoSavedSettings | ImGuiWindowFlags_NoTitleBar);
+        const int cjk_char_utf8_bytes = 3;
+        const char* s1 = "零一二三四五六七八九十百千万";   // every char must be CJK normal chars
+        const int s1_size = strlen(s1);
+        for (int i = 0; i * cjk_char_utf8_bytes  < s1_size ; ++i) {
+            const float local_off_x = ImGui::GetCursorPos().x;
+            const float wrap_width = ImGui::CalcTextSize(s1, s1 + i * cjk_char_utf8_bytes).x;
+            const char* s1_w = ImGui::GetFont()->CalcWordWrapPositionCJK(ImGui::GetFontSize(), s1, s1 + s1_size, wrap_width);
+            // printf("%p %p %p\n", s1, s1_w, s1 + ImMax(3, i));
+            IM_CHECK_EQ(s1_w, s1 + ImMax(cjk_char_utf8_bytes, i * cjk_char_utf8_bytes));
+        }
+        ImGui::End();
+    };
+    */
+
+
+    auto cjk_wrap_test_helper= [](const char * cjk_s1, const size_t * expected_wrap_offset_char, size_t expected_wrap_offset_char_count){
+        ImGui::Begin("Test Window", NULL, ImGuiWindowFlags_NoSavedSettings | ImGuiWindowFlags_NoTitleBar);
+        // common prepare, get the total chars in utf8 string and calculate the expected_wrap_position_byte
+        static size_t const max_chars = 9999;
+        size_t cbyte[max_chars];
+        size_t total_chars = ImStrUtf8CollectOffsets(cjk_s1, cjk_s1 + strlen(cjk_s1), &cbyte[0], max_chars);
+        // test wrap position for every char, so the count of expects should be equal to total_chars
+        IM_ASSERT(expected_wrap_offset_char_count == total_chars);
+        size_t expected_wrap_offset_byte[max_chars];
+        for (size_t i =0; i < total_chars; ++i){
+            expected_wrap_offset_byte[i] = cbyte[expected_wrap_offset_char[i]];
+        }
+        // test
+        ImFont* font = ImGui::FindFontByPrefix(TEST_SUITE_ALT_FONT_NAME_CJK);
+        ImGui::PushFont(font);
+        if (ImGui::BeginTable("## display", 3, ImGuiTableFlags_NoSavedSettings | ImGuiTableFlags_Borders | ImGuiTableFlags_ScrollY))
+        {
+            ImGui::TableSetupColumn("Full Text");
+            ImGui::TableSetupColumn("Result");
+            ImGui::TableSetupColumn("Explain");
+            ImGui::TableHeadersRow();
+            // test wrap position for every char!
+            for (size_t i = 0; i < total_chars; ++i)
+            {
+                float wrap_width = ImGui::CalcTextSize(cjk_s1, cjk_s1 + cbyte[i]).x;
+                const char* s1_w = ImGui::GetFont()->CalcWordWrapPositionCJK(
+                    ImGui::GetFontSize(), cjk_s1, cjk_s1 + strlen(cjk_s1), wrap_width);
+                    ImGui::TableNextColumn();
+                    ImGui::TextUnformatted(cjk_s1);
+                    ImVec2 text_pos = ImGui::GetItemRectMin();
+                    float x0 = text_pos.x;
+                    float y0 = text_pos.y;
+                    float height = ImGui::GetItemRectSize().y;
+                    float real_wrap_width =  ImGui::CalcTextSize(cjk_s1, s1_w).x;
+                    ImGui::GetWindowDrawList()->AddLine(
+                        {x0+wrap_width, y0},{x0+wrap_width, y0+height * 2},
+                        IM_COL32(255,255, 0, 255), 2.0F
+                    );
+                    ImGui::GetWindowDrawList()->AddRect(
+                        {x0, y0},{x0+real_wrap_width, y0+height},
+                        IM_COL32(0,255, 0, 255)
+                    );
+                    ImGui::TableNextColumn();
+                    ImGui::TextUnformatted(cjk_s1, s1_w);
+                    ImGui::TextUnformatted(s1_w);
+                    ImGui::TableNextColumn();
+                    ImGui::TextUnformatted(i == 0 || i == expected_wrap_offset_char[i] ? "" : "pronunciation mark rule");
+                IM_CHECK_EQ(s1_w, cjk_s1 + expected_wrap_offset_byte[i]);
+            }
+            ImGui::EndTable();
+        }
+        ImGui::PopFont();
+        ImGui::End();
+    };
+
+   // # Test CJK(zh-cn) HeadProhibitedW
+    // the wraps are not allowed to be placed before pronunciation_mark, e.g 。 》, ！ ､
+    t = IM_REGISTER_TEST(e, "widgets", "widgets_text_wrapped_cjk_zh_cn_mixed_char_with_pronunciation_mark_not_allow_at_head");
+    t->GuiFunc = [&cjk_wrap_test_helper](ImGuiTestContext* ctx)
+    {
+        // test data
+        const char cjk_s1[] = "零一，二三。四五六》七八九！十百､千万";
+        const size_t expected_wrap_offset_char[] = { 1, 1, 1, 3, 4, 4, 6, 7, 8, 8, 10, 11, 12, 12, 14, 15, 15, 17, 18 };
+        cjk_wrap_test_helper(
+            cjk_s1,
+            expected_wrap_offset_char,
+            sizeof expected_wrap_offset_char / sizeof expected_wrap_offset_char[0]);
+    };
+
+    // Test CJK(zh-cn) TailProhibitedW
+    // the wraps are not allowed to be placed after pronunciation_mark, e.g 《 【 ¥｛
+    t = IM_REGISTER_TEST(e, "widgets", "widgets_text_wrapped_cjk_zh_cn_mixed_char_with_pronunciation_mark_not_allow_at_tail");
+    t->GuiFunc = [&cjk_wrap_test_helper](ImGuiTestContext* ctx)
+    {
+        // test data
+        const char cjk_s1[] = "零一《二三【四五¥六七｛八九";
+        // based on cjk_s1 manually set the offset of expected wrap pos (not byte)
+        const size_t expected_wrap_offset_char[] = { 1, 1, 2, 2, 4, 5, 5, 7, 8, 8, 10, 11, 11, 13 };  
+        cjk_wrap_test_helper(
+            cjk_s1,
+            expected_wrap_offset_char,
+            sizeof expected_wrap_offset_char / sizeof expected_wrap_offset_char[0]);
+    };
+
+    // Test Mixed CJK(zh-cn) and English
+    // no special rule for CJK chars
+    t = IM_REGISTER_TEST(e, "widgets", "widgets_text_wrapped_cjk_zh_cn_mixed_with_english");
+    t->GuiFunc = [&cjk_wrap_test_helper](ImGuiTestContext* ctx)
+    {
+        // test data
+        const char cjk_s1[] = "零一hello world二三";  // contains CJK characters
+        // based on cjk_s1 manually set the offset of expected wrap pos (not byte)
+        const size_t expected_wrap_offset_char[] = { 1, 1, 2, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 13, 14};
+        cjk_wrap_test_helper(
+            cjk_s1,
+            expected_wrap_offset_char,
+            sizeof expected_wrap_offset_char / sizeof expected_wrap_offset_char[0]);
+    };
+
+#endif
+
     // ## Test LabelText() variants layout (#4004)
     t = IM_REGISTER_TEST(e, "widgets", "widgets_label_text");
     t->GuiFunc = [](ImGuiTestContext* ctx)

imgui_test_suite/tools/make_ttf_subset.py

@@ -0,0 +1,76 @@
+import os
+
+from fontTools import subset  # pip install fonttools
+from fontTools.ttLib import TTFont
+
+
+def create_font_subset(input_font_path, output_font_path, extra_chars="", include_ranges=None):
+    """
+    Create a font subset
+
+    Args:
+        input_font_path: Path to the input font file
+        output_font_path: Path to the output font file
+        extra_chars: Additional characters to include (as string)
+        include_ranges: List of Unicode ranges to include, defaults to basic character regions
+    """
+
+    # Default Unicode ranges to include (Basic Latin characters, numbers, punctuation, etc.)
+    if include_ranges is None:
+        include_ranges = [
+            (0x0020, 0x007F),  # Basic Latin characters
+            (0x00A0, 0x00FF),  # Latin-1 Supplement
+            (0x0100, 0x017F),  # Latin Extended-A
+            (0x2000, 0x206F),  # General Punctuation
+            (0x3000, 0x303F),  # CJK Symbols and Punctuation
+        ]
+
+    # Build Unicode parameters
+    unicode_ranges = []
+    for start, end in include_ranges:
+        unicode_ranges.append(f"U+{start:04X}-{end:04X}")
+
+    # Add extra characters
+    if extra_chars:
+        for char in extra_chars:
+            code_point = ord(char)
+            unicode_ranges.append(f"U+{code_point:04X}")
+
+    unicode_param = ",".join(unicode_ranges)
+
+    # Set subsetting options
+    options = subset.Options()
+    options.text = None
+    options.unicodes = unicode_param
+    options.passthrough_tables = True
+    options.ignore_missing_unicodes = True
+    options.ignore_missing_glyphs = True
+
+    # Execute subsetting
+    try:
+        subset.main(
+            [
+                input_font_path,
+                f"--output-file={output_font_path}",
+                f"--unicodes={unicode_param}",
+                "--passthrough-tables",
+                "--ignore-missing-unicodes",
+                "--ignore-missing-glyphs",
+            ]
+        )
+        print(f"Successfully created font subset: {output_font_path}")
+    except Exception as err:
+        print(f"Failed to create font subset: {err}")
+        return False
+    else:
+        return True
+
+
+# Usage example
+if __name__ == "__main__":
+    # Example 1: Basic usage
+    create_font_subset(
+        input_font_path="SourceHanSansCN-VF.ttf",
+        output_font_path="SourceHanSansCN-VF-subset.ttf",
+        extra_chars="零一二三四五六七八九十百千万" + "¢°′″‰℃、。｡､￠，．：；？！％・･ゝゞヽヾーァィゥェォッャュョヮヵヶぁぃぅぇぉっゃゅょゎゕゖㇰㇱㇲㇳㇴㇵㇶㇷㇸㇹㇺㇻㇼㇽㇾㇿ々〻ｧｨｩｪｫｬｭｮｯｰ”〉》」】〕）］｝｣‘“〈《「『【〔（［｛｢£¥＄￡￥＋"
+    )