diff --git a/.github/config/ubsan_suppressions.txt b/.github/config/ubsan_suppressions.txt new file mode 100644 index 00000000..c71bedce --- /dev/null +++ b/.github/config/ubsan_suppressions.txt @@ -0,0 +1,13 @@ +# UBSan/IntSan suppression file for known false positives in libstdc++ headers. +# These are intentional unsigned arithmetic patterns in the C++ standard library +# that cannot be fixed in project code. + +# std::string::npos is size_t(-1); comparisons against it use intentional unsigned wrap +unsigned-integer-overflow:*bits/basic_string.h* + +# Mersenne Twister PRNG uses deliberate unsigned modular arithmetic +unsigned-integer-overflow:*bits/random.tcc* + +# uniform_int_distribution rejection sampling uses unsigned negation (-range) +unsigned-integer-overflow:*bits/uniform_int_dist.h* +implicit-integer-sign-change:*bits/uniform_int_dist.h* diff --git a/.github/workflows/sanitizer_tests.yaml b/.github/workflows/sanitizer_tests.yaml new file mode 100644 index 00000000..7f53b7a2 --- /dev/null +++ b/.github/workflows/sanitizer_tests.yaml @@ -0,0 +1,233 @@ +name: Sanitizer Tests + +on: + schedule: + - cron: '0 18 * * *' # Daily 18:00 UTC (20:00 CEST / 19:00 CET) + workflow_dispatch: + push: + branches: ['ajanczew-sanitizers'] # TODO: remove before merge + +defaults: + run: + shell: bash + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + JOBS_NUM: 16 + +jobs: + # ── Address / Thread / Undefined / Integer sanitizers ────────────────────── + # Release build with clang; runs unit tests (parrallelUT.sh) + conformance + # (ParallelAllTests.sh) non-halting, then gates once via the summary step. + sanitizer-tests: + name: "Sanitizer: ${{ matrix.sanitizer }}" + runs-on: ['self-hosted', 'linux', 'x64', 'docker', 'kubernetes', 'valgrind', 'jpeg-perf'] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - sanitizer: address + - sanitizer: thread + - sanitizer: undefined + - sanitizer: integer + steps: + - name: 'Harden Runner' + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit + + - name: 'Setup: Cleanup workspace' + run: | + rm -rf ./* || true + rm -rf ./.??* || true + + - name: 'Checkout repository' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: 'Setup: Install dependencies' + run: | + sudo apt-get update + sudo apt-get -y install cmake nasm clang llvm make + + - name: 'Fix workspace permissions' + if: always() + run: | + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" || true + chmod -R u+rwX "$GITHUB_WORKSPACE" || true + + - name: 'Build: release with ${{ matrix.sanitizer }} sanitizer' + working-directory: Build/linux + run: | + ./build.sh release test sanitizer=${{ matrix.sanitizer }} cc=clang cxx=clang++ jobs=$(nproc) + + - name: 'Unit tests: ${{ matrix.sanitizer }} sanitizer (release)' + # continue-on-error: collect all findings; the summary step is the single gate + continue-on-error: true + env: + # Non-halting: one pass collects every finding; the summary step gates. + ASAN_OPTIONS: "halt_on_error=0:exitcode=0:print_stacktrace=1" + TSAN_OPTIONS: "halt_on_error=0:exitcode=0:history_size=4" + UBSAN_OPTIONS: "suppressions=${{ github.workspace }}/.github/config/ubsan_suppressions.txt:print_stacktrace=1:halt_on_error=0:exitcode=0" + run: | + cd Bin/Release/ + chmod +x ${{ github.workspace }}/tests/scripts/parrallelUT.sh + chmod +x ./* + export LD_LIBRARY_PATH=$(pwd) + mkdir -p "${{ github.workspace }}/sanitizer-logs" + # pipefail so a failing shard is not masked by tee + set -o pipefail + ${{ github.workspace }}/tests/scripts/parrallelUT.sh ./SvtJpegxsUnitTests ${{ env.JOBS_NUM }} \ + 2>&1 | tee "${{ github.workspace }}/sanitizer-logs/unit-tests-${{ matrix.sanitizer }}.log" + + - name: 'Conformance tests: ${{ matrix.sanitizer }} sanitizer (release)' + # continue-on-error: collect all findings; the summary step is the single gate + continue-on-error: true + working-directory: ${{ github.workspace }}/tests/scripts + env: + # Non-halting: one pass collects every finding; the summary step gates. + ASAN_OPTIONS: "halt_on_error=0:exitcode=0:print_stacktrace=1" + TSAN_OPTIONS: "halt_on_error=0:exitcode=0:history_size=4" + UBSAN_OPTIONS: "suppressions=${{ github.workspace }}/.github/config/ubsan_suppressions.txt:print_stacktrace=1:halt_on_error=0:exitcode=0" + DEC_BIN_DIR: ${{ github.workspace }}/Bin/Release + INPUT_FILES_PATH: /opt/samples + run: | + mkdir -p "${{ github.workspace }}/sanitizer-logs" + chmod +x ./*.sh + chmod +x "$DEC_BIN_DIR"/* + export LD_LIBRARY_PATH="$DEC_BIN_DIR" + # pipefail so a failing conformance shard is not masked by tee + set -o pipefail + ./ParallelAllTests.sh ${{ env.JOBS_NUM }} "$INPUT_FILES_PATH" "$DEC_BIN_DIR/SvtJpegxsDecApp" \ + 2>&1 | tee "${{ github.workspace }}/sanitizer-logs/conformance-tests-${{ matrix.sanitizer }}.log" + + - name: 'Summarize: ${{ matrix.sanitizer }} sanitizer findings' + if: always() + working-directory: ${{ github.workspace }} + run: | + chmod +x tests/scripts/sanitizer_summary.sh + # integer is report-only (does not gate on findings); every other sanitizer fails on findings. + # A non-zero test-harness exit (broken run) fails the job for ALL sanitizers, integer included. + tests/scripts/sanitizer_summary.sh ${{ matrix.sanitizer == 'integer' && '--no-gate ' || '' }}"${{ matrix.sanitizer }}" \ + sanitizer-logs/unit-tests-${{ matrix.sanitizer }}.log \ + sanitizer-logs/conformance-tests-${{ matrix.sanitizer }}.log + + - name: 'Archive: upload binary and logs on failure' + if: failure() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: sanitizer-${{ matrix.sanitizer }}-failure-${{ github.run_number }} + retention-days: 7 + path: | + Bin/Release/SvtJpegxsUnitTests + Bin/Release/SvtJpegxsEncApp + Bin/Release/SvtJpegxsDecApp + sanitizer-logs/unit-tests-${{ matrix.sanitizer }}.log + sanitizer-logs/conformance-tests-${{ matrix.sanitizer }}.log + + # Integer sanitizer runs report-only (halt_on_error=0), so the job stays + # green; always archive its logs so the findings are not lost. + - name: 'Archive: upload integer sanitizer report' + if: always() && matrix.sanitizer == 'integer' + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: sanitizer-integer-report-${{ github.run_number }} + retention-days: 7 + path: | + sanitizer-logs/unit-tests-integer.log + sanitizer-logs/conformance-tests-integer.log + + - name: 'Cleanup: Remove workspace artifacts' + if: always() + run: | + rm -rf Bin/ Build/linux/Release/ || true + + # ── Memory sanitizer ──────────────────────────────────────────────────────── + # Must be Debug (release does not build with MSan). + # Unit tests can't run under MSan: gtest aborts during global init. + # Use the C encoder/decoder apps instead. + sanitizer-memory: + name: "Sanitizer: memory" + runs-on: ['self-hosted', 'linux', 'x64', 'docker', 'kubernetes', 'valgrind', 'jpeg-perf'] + timeout-minutes: 180 + steps: + - name: 'Harden Runner' + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit + + - name: 'Setup: Cleanup workspace' + run: | + rm -rf ./* || true + rm -rf ./.??* || true + + - name: 'Checkout repository' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: 'Setup: Install dependencies' + run: | + sudo apt-get update + sudo apt-get -y install cmake nasm clang llvm make + + - name: 'Fix workspace permissions' + if: always() + run: | + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" || true + chmod -R u+rwX "$GITHUB_WORKSPACE" || true + + - name: 'Build: debug with memory sanitizer' + working-directory: Build/linux + run: | + ./build.sh debug sanitizer=memory cc=clang cxx=clang++ jobs=$(nproc) + + - name: 'Conformance tests: memory sanitizer (debug)' + # continue-on-error: collect all findings; the summary step is the single gate + continue-on-error: true + working-directory: ${{ github.workspace }}/tests/scripts + env: + DEC_BIN_DIR: ${{ github.workspace }}/Bin/Debug + INPUT_FILES_PATH: /opt/samples + # Non-halting: collect findings in one pass; the summary step gates. + MSAN_OPTIONS: "halt_on_error=0:exitcode=0" + # avx2: AVX-512 paths produce MSan false positives + SANITIZER_ASM: avx2 + run: | + mkdir -p "${{ github.workspace }}/sanitizer-logs" + chmod +x ./*.sh + chmod +x "$DEC_BIN_DIR"/* + export LD_LIBRARY_PATH="$DEC_BIN_DIR" + # pipefail so a failing conformance shard is not masked by tee + set -o pipefail + # fast: reduce the matrix (MSan is very slow); SANITIZER_ASM pins avx2 + ./ParallelAllTests.sh ${{ env.JOBS_NUM }} "$INPUT_FILES_PATH" "$DEC_BIN_DIR/SvtJpegxsDecApp" fast \ + 2>&1 | tee "${{ github.workspace }}/sanitizer-logs/memory-conformance-tests.log" + + - name: 'Summarize: memory sanitizer findings' + if: always() + working-directory: ${{ github.workspace }} + run: | + chmod +x tests/scripts/sanitizer_summary.sh + tests/scripts/sanitizer_summary.sh "memory" \ + sanitizer-logs/memory-conformance-tests.log + + - name: 'Archive: upload binaries and logs on failure' + if: failure() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: sanitizer-memory-failure-${{ github.run_number }} + retention-days: 7 + path: | + Bin/Debug/SvtJpegxsEncApp + Bin/Debug/SvtJpegxsDecApp + sanitizer-logs/memory-conformance-tests.log + + - name: 'Cleanup: Remove workspace artifacts' + if: always() + run: | + rm -rf Bin/ Build/linux/Debug/ || true diff --git a/tests/UnitTests/UnitTest_AVX512/CodeDeprecated-avx512.cc b/tests/UnitTests/UnitTest_AVX512/CodeDeprecated-avx512.cc index 86ed8467..14d88231 100644 --- a/tests/UnitTests/UnitTest_AVX512/CodeDeprecated-avx512.cc +++ b/tests/UnitTests/UnitTest_AVX512/CodeDeprecated-avx512.cc @@ -237,17 +237,18 @@ int dwt_horizontal_depricated_avx512(int32_t* out_lf, int32_t* out_hf, const int } if (remaining > 1 && simd_batch > 0) { - dst_lf_row[0] = src_row[0] + ((dst_hf_row[-1] + dst_hf_row[0] + 2) >> 2); + // Widen to int64 so the intermediate sums cannot overflow int32 (UBSan). + dst_lf_row[0] = (int32_t)((int64_t)src_row[0] + (((int64_t)dst_hf_row[-1] + dst_hf_row[0] + 2) >> 2)); } else if (remaining > 1) { // Fix: When simd_batch==0, dst_hf_row[-1] is before the allocated buffer. // Use the boundary formula instead; this value gets overwritten by // "correct the first for lf" anyway. - dst_lf_row[0] = src_row[0] + ((dst_hf_row[0] + 1) >> 1); + dst_lf_row[0] = (int32_t)((int64_t)src_row[0] + (((int64_t)dst_hf_row[0] + 1) >> 1)); } for (uint32_t i = 1; i < (remaining / 2); i++) { - dst_lf_row[i] = src_row[2 * i] + ((dst_hf_row[i - 1] + dst_hf_row[i] + 2) >> 2); + dst_lf_row[i] = (int32_t)((int64_t)src_row[2 * i] + (((int64_t)dst_hf_row[i - 1] + dst_hf_row[i] + 2) >> 2)); } dst_lf_row += remaining / 2; dst_hf_row += remaining / 2; @@ -256,7 +257,7 @@ int dwt_horizontal_depricated_avx512(int32_t* out_lf, int32_t* out_hf, const int // correct the last for hf and lf if (width % 2) { - dst_lf_row[0] = src_row[-1] + ((dst_hf_row[-1] + 1) >> 1); + dst_lf_row[0] = (int32_t)((int64_t)src_row[-1] + (((int64_t)dst_hf_row[-1] + 1) >> 1)); } else { dst_hf_row[-1] = src_row[-1] - src_row[-2]; @@ -264,7 +265,7 @@ int dwt_horizontal_depricated_avx512(int32_t* out_lf, int32_t* out_hf, const int // the buffer. Skip the LF correction here; "correct the first for lf" below // will write the correct value for LF[0]. if (width >= 4) { - dst_lf_row[-1] = src_row[-2] + ((dst_hf_row[-2] + dst_hf_row[-1] + 2) >> 2); + dst_lf_row[-1] = (int32_t)((int64_t)src_row[-2] + (((int64_t)dst_hf_row[-2] + dst_hf_row[-1] + 2) >> 2)); } } diff --git a/tests/scripts/DecoderConformanceTest.sh b/tests/scripts/DecoderConformanceTest.sh index 3395a136..2cbb3a02 100755 --- a/tests/scripts/DecoderConformanceTest.sh +++ b/tests/scripts/DecoderConformanceTest.sh @@ -42,7 +42,7 @@ function test_dec { return fi - cmd="$valgrind$exec_dec --find-bitstream-header -i $path_bitstreams/test_bitsreams/$name.jxs -o ./$tmp_dir/$yuv_name --lp $PARAM_LP_NUM --asm $PARAM_ASM --packetization-mode $PARAM_PACKETIZATION" + cmd="$valgrind$exec_dec --find-bitstream-header -i $path_bitstreams/test_bitsreams/$name.jxs -o ./$tmp_dir/$yuv_name --lp $PARAM_LP_NUM --asm ${SANITIZER_ASM:-$PARAM_ASM} --packetization-mode $PARAM_PACKETIZATION" echo "run command: $cmd" ${cmd} @@ -87,7 +87,7 @@ function test_msb_aligned_output { return fi - cmd="$valgrind$exec_dec -i $bitstream_msb_prep -o $yuv_tmp --lp $PARAM_LP_NUM --asm $PARAM_ASM --packetization-mode $PARAM_PACKETIZATION "$params + cmd="$valgrind$exec_dec -i $bitstream_msb_prep -o $yuv_tmp --lp $PARAM_LP_NUM --asm ${SANITIZER_ASM:-$PARAM_ASM} --packetization-mode $PARAM_PACKETIZATION "$params echo "run command: $cmd" ${cmd} diff --git a/tests/scripts/DecoderMultiFramesTest.sh b/tests/scripts/DecoderMultiFramesTest.sh index 6784213f..b0aa6745 100755 --- a/tests/scripts/DecoderMultiFramesTest.sh +++ b/tests/scripts/DecoderMultiFramesTest.sh @@ -51,7 +51,7 @@ function test_dec { return fi - cmd="$valgrind$exec_dec -i $bin_name -o $yuv_tmp --lp $PARAM_LP_NUM --asm $PARAM_ASM --packetization-mode $PARAM_PACKETIZATION "$params + cmd="$valgrind$exec_dec -i $bin_name -o $yuv_tmp --lp $PARAM_LP_NUM --asm ${SANITIZER_ASM:-$PARAM_ASM} --packetization-mode $PARAM_PACKETIZATION "$params echo "run command: $cmd" ${cmd} @@ -142,6 +142,10 @@ function test_all_broken { PARAM_LP_NUM=$2 PARAM_PACKETIZATION="0" + # broken_* fixtures live alongside the correct ones; set explicitly so this + # function is order-independent (in 'fast' mode test_all_correct is skipped). + path_use=$path_correct + # TEST WITH IGNORE SOME FRAMES #a. broken_decomh_Daylight_1280x720_8b_422_20fx1fx20f.jxs #1. 20frames 8bit yuv422 1280x720 v1 h5 diff --git a/tests/scripts/EncoderTest.sh b/tests/scripts/EncoderTest.sh index 90703db7..fff3857f 100755 --- a/tests/scripts/EncoderTest.sh +++ b/tests/scripts/EncoderTest.sh @@ -140,7 +140,7 @@ mkdir $tmp_dir #RUN different RC parameters Release/Debug/ ASM_C/ASM_MAX compare Parameters (1:asm) (2:lp number) function test_rate_control { - asm=$1 + asm=${SANITIZER_ASM:-$1} lp=$2 cpu_profile=$3 packetization_mode=$4 @@ -306,7 +306,7 @@ function test_rate_control { #RUN different RC parameters Release/Debug/ ASM_C/ASM_MAX compare Parameters (1:asm) (2:lp number) function test_rate_control_signs { - asm=$1 + asm=${SANITIZER_ASM:-$1} lp=$2 cpu_profile=$3 packetization_mode=$4 @@ -332,7 +332,7 @@ function test_rate_control_signs { #RUN input-msb-aligned Parameters (1:asm) (2:lp number) function test_msb_aligned { - asm=$1 + asm=${SANITIZER_ASM:-$1} lp=$2 cpu_profile=$3 packetization_mode=$4 @@ -350,7 +350,7 @@ function test_msb_aligned { } function test_uncommon_resolution { - asm=$1 + asm=${SANITIZER_ASM:-$1} lp=$2 cpu_profile=$3 packetization_mode=$4 @@ -634,7 +634,7 @@ function test_uncommon_resolution { } function test_handle_errors { - asm=$1 + asm=${SANITIZER_ASM:-$1} lp=$2 cpu_profile=$3 packetization_mode=$4 @@ -721,7 +721,7 @@ function test_handle_errors { #RUN different RC parameters Release/Debug/ ASM_C/ASM_MAX compare Parameters (1:asm) (2:lp number) function test_invalid_yuv { - asm=$1 + asm=${SANITIZER_ASM:-$1} lp=$2 cpu_profile=$3 packetization_mode=$4 diff --git a/tests/scripts/sanitizer_summary.sh b/tests/scripts/sanitizer_summary.sh new file mode 100755 index 00000000..81d093e2 --- /dev/null +++ b/tests/scripts/sanitizer_summary.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# +# Copyright(c) 2025 Intel Corporation +# SPDX - License - Identifier: BSD - 2 - Clause - Patent +# +# Summarize sanitizer findings from one or more run logs into a Markdown report +# (written to $GITHUB_STEP_SUMMARY when set, otherwise stdout). Used by the +# sanitizer CI jobs, which run non-halting (halt_on_error=0) so a single pass +# collects every finding instead of stopping at the first one. +# +# Used as the job's final gate: +# * any sanitizer finding -> fail (unless --no-gate, e.g. integer) +# * run did not complete cleanly -> fail always (never a silent green) +# * clean + complete -> pass +# +# Usage: sanitizer_summary.sh [--no-gate]