build: fix cmake dependencies, sdist self-containment, and wheel CI w…

.github/workflows/cross_build.yml

@@ -30,9 +30,9 @@ jobs:
     - name: Install cross tools
       run: |
         sudo apt-get update
-        sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump {gcc,g++}-10-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4,alpha}-linux-gnu {gcc,g++}-10-arm-linux-gnueabihf
-        sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 /usr/bin/arm-linux-gnu-gcc-10
-        sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 /usr/bin/arm-linux-gnu-g++-10
+        sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump {gcc,g++}-14-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4,alpha}-linux-gnu {gcc,g++}-14-arm-linux-gnueabihf
+        sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-14 /usr/bin/arm-linux-gnu-gcc-14
+        sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-14 /usr/bin/arm-linux-gnu-g++-14
         sudo ln -sf /usr/arm-linux-gnueabihf /usr/arm-linux-gnu
 
     - name: Build
@@ -42,7 +42,7 @@ jobs:
         mkdir -p build
         cd build
         qemu_arch=`echo "$ARCH" | sed -e s/powerpc/ppc/ -e s/686/386/`
-        env CXX=/usr/bin/${ARCH}-linux-gnu-g++-10 CC=/usr/bin/${ARCH}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${ARCH}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_CROSSCOMPILING_EMULATOR="qemu-${qemu_arch};-L;/usr/${ARCH}-linux-gnu"
+        env CXX=/usr/bin/${ARCH}-linux-gnu-g++-14 CC=/usr/bin/${ARCH}-linux-gnu-gcc-14 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${ARCH}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_CROSSCOMPILING_EMULATOR="qemu-${qemu_arch};-L;/usr/${ARCH}-linux-gnu"
         make -j$(nproc)
 
     - name: Test on QEMU

.github/workflows/wheel.yml

@@ -82,20 +82,22 @@
         run: |
           mkdir -p src/sentencepiece/package_data
           cp ../data/*.bin src/sentencepiece/package_data
+          python build_sdist.py
           python -m cibuildwheel --output-dir wheelhouse
         env:
           CIBW_ARCHS_LINUX: auto
           CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }}
           CIBW_ARCHS_WINDOWS: auto
           CIBW_SKIP: "*-musllinux_* *-win32"
           CIBW_ENVIRONMENT: "CMAKE_BUILD_PARALLEL_LEVEL=8"
+          CIBW_BEFORE_ALL_LINUX: "sh build_bundled.sh"
           CIBW_BUILD_VERBOSITY: 1
 
       - name: Build sdist archive
         if: matrix.os == 'ubuntu-latest'
         working-directory: python
         run: |
-          sh build_sdist.sh
+          python build_sdist.py
 
       - name: Fetch sdist archive
         if: matrix.os == 'ubuntu-latest'
@@ -132,7 +134,7 @@
 
       - name: Upload wheel release
         if: startsWith(github.ref, 'refs/tags/')
         uses: svenstaro/upload-release-action@29e53e917877a24fad85510ded594ab3c9ca12de # v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: python/wheelhouse/*

CMakeLists.txt

@@ -277,7 +277,6 @@ if (SPM_PROTOBUF_PROVIDER STREQUAL "module")
     endif()
   endif()
   FetchContent_GetProperties(protobuf SOURCE_DIR protobuf_SOURCE_DIR)
-  include_directories(${protobuf_SOURCE_DIR}/src ${protobuf_SOURCE_DIR}/third_party/utf8_range)
 endif()
 
 if(SPM_ENABLE_BENCHMARK)

python/build_bundled.sh

@@ -1,7 +1,5 @@
 #!/bin/sh
 
-VERSION="$1"
-
 mkdir -p build
 
 BUILD_DIR=./build
@@ -12,10 +10,8 @@ if [ -f ./sentencepiece/src/CMakeLists.txt ]; then
 elif [ -f ../src/CMakeLists.txt ]; then
   SRC_DIR=..  
 else
-  # Try tagged version. Othewise, use head.
-  git clone https://github.com/google/sentencepiece.git -b v"${VERSION}" --depth 1 || \
-  git clone https://github.com/google/sentencepiece.git --depth 1
-  SRC_DIR=./sentencepiece
+  echo "Error: SentencePiece C++ source files not found in ./sentencepiece or ../" >&2
+  exit 1
 fi
 
 NPROC=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)

python/build_sdist.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import subprocess
+import sys
+
+# Copy C++ source files from parent directory to ./sentencepiece/
+targets = [
+    'CMakeLists.txt',
+    'LICENSE',
+    'README.md',
+    'VERSION.txt',
+    'cmake',
+    'config.h.in',
+    'sentencepiece.pc.in',
+    'src',
+    'third_party',
+    'data',
+]
+
+os.makedirs('sentencepiece', exist_ok=True)
+
+for item in targets:
+  src = os.path.join('..', item)
+  dst = os.path.join('sentencepiece', item)
+  if os.path.lexists(src):
+    if os.path.islink(src):
+      # Skip build-generated symlinks (e.g. third_party/absl)
+      continue
+    print(f'copying {src} -> {dst}')
+    if os.path.isdir(src):
+      shutil.copytree(
+          src,
+          dst,
+          dirs_exist_ok=True,
+          ignore=shutil.ignore_patterns('absl', '*.pyc', '__pycache__'),
+      )
+    else:
+      shutil.copy2(src, dst)
+
+python_exe = sys.executable
+res = subprocess.run([python_exe, '-m', 'build', '--sdist'], check=False)
+if res.returncode != 0:
+  subprocess.check_call([python_exe, 'setup.py', 'sdist'])
+

python/build_sdist.sh

@@ -1,11 +1,5 @@
 #!/bin/sh
 
-mkdir -p sentencepiece
+PYTHON="${PYTHON:-python3}"
+${PYTHON} build_sdist.py "$@"
 
-for i in CMakeLists.txt LICENSE README.md VERSION.txt cmake config.h.in sentencepiece.pc.in src third_party
-do
-  echo "copying ../${i} sentencepiece/${i}"
-  cp -f -R "../${i}" sentencepiece
-done
-
-python -m build --sdist

python/setup.py

@@ -109,7 +109,11 @@ def build_extension(self, ext):
     abseil_libs = find_abseil_lib('../build')
 
     if len(libs) == 0:
-      subprocess.check_call(['./build_bundled.sh', __version__])
+      cflags, libs = get_cflags_and_libs('./build')
+      abseil_libs = find_abseil_lib('./build')
+
+    if len(libs) == 0:
+      subprocess.check_call(['./build_bundled.sh'])
       cflags, libs = get_cflags_and_libs('./build')
       abseil_libs = find_abseil_lib('./build')
 
@@ -170,6 +174,8 @@ def build_extension(self, ext):
       build_dir = '..\\build_{}'.format(arch)
     elif os.path.exists('..\\build\\root\\lib'):
       build_dir = '..\\build'
+    elif os.path.exists('.\\build\\root\\lib'):
+      build_dir = '.\\build'
     else:
       # build library locally with cmake and vc++.
       if arch == 'amd64':
@@ -244,6 +250,7 @@ def find_targets(roots):
       '../build/root/share/sentencepiece',
       './build/root/share/sentencepiece',
       '../data',
+      './sentencepiece/data',
   ])
 
   for filename in data:

src/CMakeLists.txt

@@ -27,8 +27,10 @@ if (SPM_PROTOBUF_PROVIDER STREQUAL "module")
   FetchContent_GetProperties(protobuf SOURCE_DIR protobuf_SOURCE_DIR)
   set(PROTOBUF_LITE_LIBRARY protobuf::libprotobuf-lite)
   list(APPEND SPM_LIBS ${PROTOBUF_LITE_LIBRARY})
+  if (TARGET utf8_validity)
+    list(APPEND SPM_LIBS utf8_validity)
+  endif()
   set(PROTOBUF_LITE_SRCS "")
-  include_directories(${protobuf_SOURCE_DIR}/src ${protobuf_SOURCE_DIR}/third_party/utf8_range)
   include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
   if (SPM_PROTOC_EXECUTABLE)
@@ -230,8 +232,8 @@ endif()
 add_library(sentencepiece-static STATIC ${SPM_SRCS})
 add_library(sentencepiece_train-static STATIC ${SPM_TRAIN_SRCS})
 
-target_link_libraries(sentencepiece-static INTERFACE ${SPM_LIBS})
-target_link_libraries(sentencepiece_train-static INTERFACE sentencepiece-static ${SPM_LIBS})
+target_link_libraries(sentencepiece-static PUBLIC ${SPM_LIBS})
+target_link_libraries(sentencepiece_train-static PUBLIC sentencepiece-static ${SPM_LIBS})
 add_dependencies(sentencepiece_train-static sentencepiece-static)
 
 if (SPM_ENABLE_SHARED)