From 4e5215ed86cc08d130d937c346a783c936182b0a Mon Sep 17 00:00:00 2001 From: Taku Kudo Date: Wed, 12 Aug 2026 01:50:06 +0000 Subject: [PATCH] build: fix cmake dependencies, sdist self-containment, and wheel CI workflow --- .github/workflows/cross_build.yml | 8 +++--- .github/workflows/wheel.yml | 4 ++- CMakeLists.txt | 1 - python/build_bundled.sh | 8 ++---- python/build_sdist.py | 45 +++++++++++++++++++++++++++++++ python/build_sdist.sh | 10 ++----- python/setup.py | 9 ++++++- src/CMakeLists.txt | 8 +++--- 8 files changed, 69 insertions(+), 24 deletions(-) create mode 100644 python/build_sdist.py diff --git a/.github/workflows/cross_build.yml b/.github/workflows/cross_build.yml index 8362c0184..14c084684 100644 --- a/.github/workflows/cross_build.yml +++ b/.github/workflows/cross_build.yml @@ -30,9 +30,9 @@ jobs: - name: Install cross tools run: | sudo apt-get update - sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump {gcc,g++}-10-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4,alpha}-linux-gnu {gcc,g++}-10-arm-linux-gnueabihf - sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 /usr/bin/arm-linux-gnu-gcc-10 - sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 /usr/bin/arm-linux-gnu-g++-10 + sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump {gcc,g++}-14-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4,alpha}-linux-gnu {gcc,g++}-14-arm-linux-gnueabihf + sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-14 /usr/bin/arm-linux-gnu-gcc-14 + sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-14 /usr/bin/arm-linux-gnu-g++-14 sudo ln -sf /usr/arm-linux-gnueabihf /usr/arm-linux-gnu - name: Build @@ -42,7 +42,7 @@ jobs: mkdir -p build cd build qemu_arch=`echo "$ARCH" | sed -e s/powerpc/ppc/ -e s/686/386/` - env CXX=/usr/bin/${ARCH}-linux-gnu-g++-10 CC=/usr/bin/${ARCH}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${ARCH}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_CROSSCOMPILING_EMULATOR="qemu-${qemu_arch};-L;/usr/${ARCH}-linux-gnu" + env CXX=/usr/bin/${ARCH}-linux-gnu-g++-14 CC=/usr/bin/${ARCH}-linux-gnu-gcc-14 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${ARCH}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_CROSSCOMPILING_EMULATOR="qemu-${qemu_arch};-L;/usr/${ARCH}-linux-gnu" make -j$(nproc) - name: Test on QEMU diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 142bc2721..f7b1a34e5 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -82,6 +82,7 @@ jobs: run: | mkdir -p src/sentencepiece/package_data cp ../data/*.bin src/sentencepiece/package_data + python build_sdist.py python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS_LINUX: auto @@ -89,13 +90,14 @@ jobs: CIBW_ARCHS_WINDOWS: auto CIBW_SKIP: "*-musllinux_* *-win32" CIBW_ENVIRONMENT: "CMAKE_BUILD_PARALLEL_LEVEL=8" + CIBW_BEFORE_ALL_LINUX: "sh build_bundled.sh" CIBW_BUILD_VERBOSITY: 1 - name: Build sdist archive if: matrix.os == 'ubuntu-latest' working-directory: python run: | - sh build_sdist.sh + python build_sdist.py - name: Fetch sdist archive if: matrix.os == 'ubuntu-latest' diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e0b6e60b..35806dd18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,7 +277,6 @@ if (SPM_PROTOBUF_PROVIDER STREQUAL "module") endif() endif() FetchContent_GetProperties(protobuf SOURCE_DIR protobuf_SOURCE_DIR) - include_directories(${protobuf_SOURCE_DIR}/src ${protobuf_SOURCE_DIR}/third_party/utf8_range) endif() if(SPM_ENABLE_BENCHMARK) diff --git a/python/build_bundled.sh b/python/build_bundled.sh index 7d496c358..5c6f0dd3f 100755 --- a/python/build_bundled.sh +++ b/python/build_bundled.sh @@ -1,7 +1,5 @@ #!/bin/sh -VERSION="$1" - mkdir -p build BUILD_DIR=./build @@ -12,10 +10,8 @@ if [ -f ./sentencepiece/src/CMakeLists.txt ]; then elif [ -f ../src/CMakeLists.txt ]; then SRC_DIR=.. else - # Try tagged version. Othewise, use head. - git clone https://github.com/google/sentencepiece.git -b v"${VERSION}" --depth 1 || \ - git clone https://github.com/google/sentencepiece.git --depth 1 - SRC_DIR=./sentencepiece + echo "Error: SentencePiece C++ source files not found in ./sentencepiece or ../" >&2 + exit 1 fi NPROC=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) diff --git a/python/build_sdist.py b/python/build_sdist.py new file mode 100644 index 000000000..ba134bda2 --- /dev/null +++ b/python/build_sdist.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import os +import shutil +import subprocess +import sys + +# Copy C++ source files from parent directory to ./sentencepiece/ +targets = [ + 'CMakeLists.txt', + 'LICENSE', + 'README.md', + 'VERSION.txt', + 'cmake', + 'config.h.in', + 'sentencepiece.pc.in', + 'src', + 'third_party', + 'data', +] + +os.makedirs('sentencepiece', exist_ok=True) + +for item in targets: + src = os.path.join('..', item) + dst = os.path.join('sentencepiece', item) + if os.path.lexists(src): + if os.path.islink(src): + # Skip build-generated symlinks (e.g. third_party/absl) + continue + print(f'copying {src} -> {dst}') + if os.path.isdir(src): + shutil.copytree( + src, + dst, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns('absl', '*.pyc', '__pycache__'), + ) + else: + shutil.copy2(src, dst) + +python_exe = sys.executable +res = subprocess.run([python_exe, '-m', 'build', '--sdist'], check=False) +if res.returncode != 0: + subprocess.check_call([python_exe, 'setup.py', 'sdist']) + diff --git a/python/build_sdist.sh b/python/build_sdist.sh index 76d795f96..1c907cf04 100755 --- a/python/build_sdist.sh +++ b/python/build_sdist.sh @@ -1,11 +1,5 @@ #!/bin/sh -mkdir -p sentencepiece +PYTHON="${PYTHON:-python3}" +${PYTHON} build_sdist.py "$@" -for i in CMakeLists.txt LICENSE README.md VERSION.txt cmake config.h.in sentencepiece.pc.in src third_party -do - echo "copying ../${i} sentencepiece/${i}" - cp -f -R "../${i}" sentencepiece -done - -python -m build --sdist diff --git a/python/setup.py b/python/setup.py index c303be4ce..0ecb17e50 100755 --- a/python/setup.py +++ b/python/setup.py @@ -109,7 +109,11 @@ def build_extension(self, ext): abseil_libs = find_abseil_lib('../build') if len(libs) == 0: - subprocess.check_call(['./build_bundled.sh', __version__]) + cflags, libs = get_cflags_and_libs('./build') + abseil_libs = find_abseil_lib('./build') + + if len(libs) == 0: + subprocess.check_call(['./build_bundled.sh']) cflags, libs = get_cflags_and_libs('./build') abseil_libs = find_abseil_lib('./build') @@ -170,6 +174,8 @@ def build_extension(self, ext): build_dir = '..\\build_{}'.format(arch) elif os.path.exists('..\\build\\root\\lib'): build_dir = '..\\build' + elif os.path.exists('.\\build\\root\\lib'): + build_dir = '.\\build' else: # build library locally with cmake and vc++. if arch == 'amd64': @@ -244,6 +250,7 @@ def find_targets(roots): '../build/root/share/sentencepiece', './build/root/share/sentencepiece', '../data', + './sentencepiece/data', ]) for filename in data: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3f61bc23c..475778aae 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,8 +27,10 @@ if (SPM_PROTOBUF_PROVIDER STREQUAL "module") FetchContent_GetProperties(protobuf SOURCE_DIR protobuf_SOURCE_DIR) set(PROTOBUF_LITE_LIBRARY protobuf::libprotobuf-lite) list(APPEND SPM_LIBS ${PROTOBUF_LITE_LIBRARY}) + if (TARGET utf8_validity) + list(APPEND SPM_LIBS utf8_validity) + endif() set(PROTOBUF_LITE_SRCS "") - include_directories(${protobuf_SOURCE_DIR}/src ${protobuf_SOURCE_DIR}/third_party/utf8_range) include_directories(${CMAKE_CURRENT_BINARY_DIR}) if (SPM_PROTOC_EXECUTABLE) @@ -230,8 +232,8 @@ endif() add_library(sentencepiece-static STATIC ${SPM_SRCS}) add_library(sentencepiece_train-static STATIC ${SPM_TRAIN_SRCS}) -target_link_libraries(sentencepiece-static INTERFACE ${SPM_LIBS}) -target_link_libraries(sentencepiece_train-static INTERFACE sentencepiece-static ${SPM_LIBS}) +target_link_libraries(sentencepiece-static PUBLIC ${SPM_LIBS}) +target_link_libraries(sentencepiece_train-static PUBLIC sentencepiece-static ${SPM_LIBS}) add_dependencies(sentencepiece_train-static sentencepiece-static) if (SPM_ENABLE_SHARED)